#!/usr/bin/env python3 """ install external stuff for dumux """ import os import shutil import re import urllib.request import tarfile import sys import subprocess import argparse import textwrap # pylint: disable=C0103,W0212,W0622,C0116 class ChoicesAction(argparse._StoreAction): """Action to show choices in argparse""" def __init__(self, **kwargs): super().__init__(**kwargs) if self.choices is None: self.choices = [] self._choices_actions = [] def add_choice(self, choice, help=""): self.choices.append(choice) choice_action = argparse.Action(option_strings=[], dest=choice, help=help) self._choices_actions.append(choice_action) def _get_subactions(self): return self._choices_actions # pylint: enable=C0103,W0212,W0622,C0116 def showMessage(message): """Pretty print for info MESSAGES""" print("*" * 120) print(message) print("") print("*" * 120) if len(sys.argv) == 1: showMessage( "No options given. For more information " "run the following command: \n ./installexternal.py --help" ) sys.exit() parser = argparse.ArgumentParser( prog="installexternal", usage="./installexternal.py [OPTIONS] PACKAGES", description="This script downloads extenstions for dumux and dune \ and installs some External Libraries and Modules.", ) parser.register("action", "store_choice", ChoicesAction) # Positional arguments group = parser.add_argument_group(title="your choice of packages") packages = group.add_argument("packages", nargs="+", metavar="PACKAGES", action="store_choice") packages.add_choice("dumux-extensions", help="Download dumux-course and dumux-lecture.") packages.add_choice( "dune-extensions", help="Download dune-uggrid, dune-alugrid, dune-foamgrid, \ dune-subgrid, dune-spgrid, dune-mmesh and dune-functions.", ) packages.add_choice("optimization", help="Download and install glpk and nlopt.") packages.add_choice("others", help="Download and install opm , metis and gstat.") packages.add_choice("lecture", help="Download dumux-lecture.") packages.add_choice("course", help="Download dumux-course.") packages.add_choice("ug", help="Download dune-uggrid.") packages.add_choice("alugrid", help="Download dune-alugrid.") packages.add_choice("foamgrid", help="Download dune-foamgrid.") packages.add_choice("subgrid", help="Download dune-subgrid.") packages.add_choice("spgrid", help="Download dune-spgrid.") packages.add_choice("mmesh", help="Download dune-mmesh.") packages.add_choice("functions", help="Download dune-functions.") packages.add_choice("glpk", help="Download and install glpk.") packages.add_choice("nlopt", help="Download and install nlopt.") packages.add_choice("opm", help="Download opm modules required for cornerpoint grids.") packages.add_choice("metis", help="Download and install the METIS graph partitioner.") packages.add_choice("gstat", help="Download and install gstat.") # Optional arguments options = parser.add_mutually_exclusive_group(required=False) options.add_argument( "--clean", action="store_true", default=False, help="Delete all files for the given packages." ) options.add_argument( "--download", action="store_true", default=False, help="Only download the packages." ) parser.add_argument("--duneBranch", default="releases/2.7", help="Dune branch to be checked out.") parser.add_argument("--dumuxBranch", default="releases/3.4", help="Dumux branch to be checked out.") parser.add_argument("--opmBranch", default="release/2020.10", help="Opm branch to be checked out.") parser.add_argument("--mmeshBranch", default="release/1.2", help="Mmesh branch to be checked out.") args = vars(parser.parse_args()) def runCommand(command, currentDir="."): """Helper function to run commands with error checking and reporting""" with open(currentDir + "/installexternal.log", "a") as log: with subprocess.Popen(command, stdout=log, stderr=log, universal_newlines=True) as popen: returnCode = popen.wait() if returnCode: message = textwrap.dedent( f""" (Error) The command {command} returned with non-zero exit code If you can't fix the problem yourself consider reporting your issue on the mailing list (dumux@listserv.uni-stuttgart.de) and attach the file 'installexternal.log' """ ) showMessage(message) sys.exit(1) def gitClone(url, branch=None): """Clone a repository from a given URL""" clone = ["git", "clone"] if branch: clone += ["-b", branch] runCommand(command=[*clone, url]) def branchName(package, parameters): """Get the correct branch name""" # Set the branch if "dumux" in package: return parameters["dumux_branch"] if "mmesh" in package: return parameters["mmesh_branch"] if "dune" in package: return parameters["dune_branch"] if "opm" in package: return parameters["opm_branch"] return "" def cleanPackage(package, finalMessage): """Clean up after a package""" if os.path.isfile(package + ".tar.gz"): os.remove(package + ".tar.gz") if os.path.exists(package): shutil.rmtree(package) finalMessage.append("{} has been removed.".format(package)) else: # Save message to be shown at the end finalMessage.append("The folder {} does not exist.".format(package)) def filterPackageList(packageListOld): """Filter the package list and add possible dependencies""" packageList = [] for pkg in packageListOld: if pkg in PACKAGE_NAMES: packageList.extend(PACKAGE_NAMES[pkg]) else: packageList.extend([key for key in EXTERNAL_URLS if pkg in key]) return packageList def checkLocation(): """check call location of this script""" if not os.path.isdir("dune-common"): showMessage( "You have to call " + sys.argv[0] + " for " + sys.argv[1] + " from\n" "the same directory in which dune-common is located.\n" "You cannot install it in this folder." ) raise Exception("Script called in wrong location. Aborting.") def installFromTarball(package, parameters, externalDir, finalMessage): """Install a package that uses a tarball as source code archive""" # Download the tarfile with urllib.request.urlopen(EXTERNAL_URLS[package]) as fileData: dataToWrite = fileData.read() with open(externalDir + "/" + package + ".tar.gz", "wb") as file: file.write(dataToWrite) # Save message to be shown at the end finalMessage.append("{} has been successfully downloaded.".format(package)) # Start Installation if the flag download is set to false. if not parameters["download"]: # Extract with tarfile.open(package + ".tar.gz") as tarArchive: tarArchive.extractall() shutil.move(os.path.commonprefix(tarArchive.getnames()), package) # rename # Start the configuration os.chdir(externalDir + "/" + package) if package == "gstat": with open("configure", "r+") as file: content = file.read() file.seek(0) file.truncate() file.write(content.replace("doc/tex/makefile", "")) # Run Configuration command configCmd = "./configure" if package != "metis" else ["make", "config"] runCommand(configCmd, currentDir=externalDir) try: runCommand("make", currentDir=externalDir) except subprocess.CalledProcessError as exc: raise Exception("{} installation has failed.".format(package)) from exc # Save message to be shown at the end if os.path.exists(externalDir + "/" + package): finalMessage.append("{} has been successfully installed.".format(package)) def installExternal(parameters): """Main driver: install external packages""" topDir = os.getcwd() externalDir = topDir + "/external" parameters["packages"] = filterPackageList(parameters["packages"]) # print the list of packages to be downloaded/installed/removed print( "The following package(s) will be {0}:\n".format( "removed" if parameters["clean"] else "downloaded" ), ", ".join(parameters["packages"]), "\n", ) checkLocation() # clear the log file logDir = externalDir if os.path.exists(externalDir) else topDir with open(logDir + "/installexternal.log", "w") as _: pass finalMessage = [] for package in parameters["packages"]: os.chdir(topDir) # Package name for final message finalMessage.append("[---" + package + "---]") # Set the directory: create externalDir for external packages if not any(re.compile(p).match(package) for p in ["dumux", "dune", "opm"]): os.makedirs(externalDir, exist_ok=True) os.chdir(externalDir) branch = branchName(package, parameters) # Run the requested command if parameters["clean"]: cleanPackage(package, finalMessage) continue # Check if tarball tarball = EXTERNAL_URLS[package].endswith("tar.gz") if not os.path.exists(package): if tarball: installFromTarball(package, parameters, externalDir, finalMessage) else: # Clone from repo gitClone(EXTERNAL_URLS[package], branch) # Save message to be shown at the end finalMessage.append("{} has been sucessfully cloned.".format(package)) else: if tarball: finalMessage.append("{} has been already installed.".format(package)) else: # Checkout to the requested branch os.chdir(topDir + "/" + package) with subprocess.Popen(["git", "checkout", branch]) as _: # Save message to be shown at the end finalMessage.append( "-- Skip cloning {}, because the folder already exists.".format(package) ) finalMessage.append("-- Checking out {} ".format(package) + branch) continue # Save post installation message if there is any. if package in MESSAGES.keys(): finalMessage.extend(MESSAGES[package]) # Change to topDir os.chdir(topDir) # Save post installation message about dunecontrol if need be. if not parameters["clean"] and any( x in pkg for pkg in parameters["packages"] for x in ["dumux", "dune", "opm"] ): finalMessage.append( "\n\nPlease run the following command " "(can be copied to command line):\n\n " "./dune-common/bin/dunecontrol --opts=./dumux/cmake.opts all" ) # If cleanup and only logfile in the external directory, remove the directory if os.path.isdir(externalDir): _, _, files = next(os.walk(externalDir)) if parameters["clean"] and len(files) == 1 and "installexternal.log" in files: shutil.rmtree(externalDir) return "\n".join(finalMessage) ################################################################# ################################################################# # (1/2) Define the necessary packages and their URLS ################################################################# ################################################################# DUNE_GIT_BASEURL = "https://gitlab.dune-project.org/" DUMUX_GIT_BASEURL = "https://git.iws.uni-stuttgart.de/dumux-repositories/" EXTERNAL_URLS = { "dumux-lecture": DUMUX_GIT_BASEURL + "dumux-lecture.git", "dumux-course": DUMUX_GIT_BASEURL + "dumux-course.git", "dune-uggrid": DUNE_GIT_BASEURL + "/staging/dune-uggrid.git", "dune-alugrid": DUNE_GIT_BASEURL + "extensions/dune-alugrid.git", "dune-foamgrid": DUNE_GIT_BASEURL + "extensions/dune-foamgrid.git", "dune-subgrid": "https://git.imp.fu-berlin.de/agnumpde/dune-subgrid.git", "dune-spgrid": DUNE_GIT_BASEURL + "extensions/dune-spgrid.git", "dune-mmesh": DUNE_GIT_BASEURL + "samuel.burbulla/dune-mmesh.git", "dune-functions": DUNE_GIT_BASEURL + "staging/dune-functions.git", "dune-typetree": DUNE_GIT_BASEURL + "staging/dune-typetree.git", "glpk": "http://ftp.gnu.org/gnu/glpk/glpk-4.60.tar.gz", "nlopt": "http://ab-initio.mit.edu/nlopt/nlopt-2.4.2.tar.gz", "opm-common": "https://github.com/OPM/opm-common", "opm-grid": "https://github.com/OPM/opm-grid", "metis": "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz", "gstat": "http://gstat.org/gstat.tar.gz", } PACKAGE_NAMES = { "dumux-extensions": ["dumux-lecture", "dumux-course"], "dune-extensions": [ "dune-uggrid", "dune-alugrid", "dune-foamgrid", "dune-subgrid", "dune-spgrid", "dune-mmesh", "dune-functions", "dune-typetree", ], "functions": ["dune-functions", "dune-typetree"], "optimization": ["glpk", "nlopt"], "others": ["opm-common", "opm-grid", "metis", "gstat"], } MESSAGES = { "glpk": [ "In addition, it might be necessary to set manually", "the glpk path in the CMAKE_FLAGS section of the .opts-file:", " -DGLPK_ROOT=/path/to/glpk \\", ], "dune-mmesh": [ "Maybe you also have to install CGAL", "(see cgal.org/download.html)", "Maybe you also need to change your core dune modules' branches to their newest versions.", ], "opm-common": [ "In addition, it might be necessary to set manually some", "CMake variables in the CMAKE_FLAGS section of the .opts-file:", " -DUSE_MPI=ON", "Currently, compiling opm with clang is not possible.", "", "Maybe you also have to install the following packages (see the", " opm prerequisites at opm-project.org):", " BLAS, LAPACK, Boost, SuiteSparse, Zoltan", ], } ################################################################# ################################################################# # (2/2) Download/config/clean the requested packages ################################################################# ################################################################# # Start download/configuration/cleaning tasks showMessage(installExternal(args))