diff --git a/bin/create_cmakelists.py b/bin/create_cmakelists.py
new file mode 100755
index 0000000000000000000000000000000000000000..3a8d4d0fe387fc7c904508e39401c099138a08b0
--- /dev/null
+++ b/bin/create_cmakelists.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+"""
+Create files CMakeLists.txt for the given folder and all subfolders,
+including the add_subdirectory(...) and install(...) commands.
+Defaults to the folder `dumux` that contains the header files,
+if no folder was specified.
+"""
+
+import os
+import argparse
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('folder', type=str, nargs='?', help='the folder to create CMakeLists.txt\'s for', default=None)
+ args = vars(parser.parse_args())
+
+ # default to the dumux folder (relative path to the location of this script)
+ if args['folder'] is None:
+ rootDir = os.path.dirname(os.path.abspath(__file__)) + "/../../dumux"
+ else:
+ rootDir = args['folder']
+
+
+ ignore_folders = ["", "io/format/fmt", "io/xml"]
+ extensions = [".hh", ".inc"]
+ for fullFolderName, subFolders, files in os.walk(rootDir):
+ # alphabetically sort
+ subFolders = sorted(subFolders)
+ files = sorted(files)
+ # get folder name relative to dumux
+ folderName = fullFolderName.replace(rootDir + '/', '').replace(rootDir, '')
+ if folderName not in ignore_folders:
+ with open(fullFolderName + "/CMakeLists.txt", "w") as cmakelists:
+ # add subfolders
+ for subFolder in subFolders:
+ cmakelists.write("add_subdirectory({})\n".format(subFolder))
+
+ headersExist = False
+ for fileName in files:
+ ext = os.path.splitext(fileName)[1]
+ if ext in extensions:
+ headersExist = True
+ break
+
+ if headersExist:
+ if subFolders: cmakelists.write("\n")
+ # collect all files to be installed in a CMake variable
+ headers_variable = "DUMUX_" + folderName.upper().replace("/", "_") + "_HEADERS"
+ cmakelists.write("file(GLOB {}{})\n".format(headers_variable, " *".join([''] + extensions)))
+ cmakelists.write("install(FILES ${{{}}}\n".format(headers_variable))
+ cmakelists.write(" DESTINATION ${{CMAKE_INSTALL_INCLUDEDIR}}/dumux/{})\n".format(folderName))
diff --git a/bin/create_dockerimage.py b/bin/create_dockerimage.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed3d002631ff7cf4509d72290730b16dbf7170c4
--- /dev/null
+++ b/bin/create_dockerimage.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+import os
+import sys
+import string
+import shutil
+import argparse
+import subprocess
+from util.moduleinfo import getModuleFile
+from util.moduleinfo import extractModuleInfos
+
+# require python 3
+if sys.version_info[0] < 3:
+ sys.exit("\nERROR: Python3 required")
+
+if __name__ == "__main__":
+
+ # input argument parser
+ parser = argparse.ArgumentParser(
+ description="Create a docker image for a given module and install script."
+ )
+
+ parser.add_argument('-m', '--modulepath',
+ required=True,
+ help='the path to the your module')
+ parser.add_argument('-i', '--installScript',
+ required=True,
+ help="Specify the installation script")
+ parser.add_argument('-t', '--templateFolder',
+ required=False,
+ help="Specify the folder with the template files")
+
+ args = vars(parser.parse_args())
+
+ # get information on the module
+ modulePath = os.path.abspath(args['modulepath'])
+ modInfo = extractModuleInfos(getModuleFile(modulePath),
+ ['Module', 'Maintainer'])
+ moduleName = modInfo['Module']
+ moduleMaintainer = modInfo['Maintainer']
+ dockerTag = moduleName.lower() # docker only supports lower case
+
+ # get folder with the template files
+ templateFolder = args['templateFolder']
+ if not templateFolder:
+ templateFolder = os.path.join(modulePath, '../dumux/docker')
+ if not os.path.exists(templateFolder):
+ sys.exit("Template folder {} could not be found".format(templateFolder))
+
+ print("*"*54)
+ print("\n-- Creating a Docker image for module " + moduleName + " --\n")
+ print("*"*54)
+
+ if os.path.exists("docker"):
+ print("\nA docker folder already exists. "
+ "Continue anyway? - will be overwritten - [y/N]\n")
+ delete = input()
+ if delete == "y" or delete == "Y":
+ shutil.rmtree("docker")
+ print("--> Deleted old docker folder.")
+ else:
+ sys.exit("Abort.")
+
+ os.mkdir("docker")
+ print("--> Created the folder 'docker'.")
+
+ # copy install script into docker folder and make it executable
+ installScriptPath = args['installScript']
+ installScriptName = os.path.split(installScriptPath)[1]
+ installScript = os.path.join(os.path.join(os.getcwd(), 'docker'),
+ installScriptName)
+ shutil.copy(installScriptPath, installScript)
+ os.system("chmod +x {}".format(installScript))
+ print("--> Using install script: {} to install dependencies for module {}."
+ .format(installScript, moduleName))
+
+
+ # substitute content from template and write to target
+ def substituteAndWrite(template, target, mapping):
+ if not os.path.exists(template):
+ sys.exit("Template file '" + template + "' could not be found")
+ with open(target, 'w') as targetFile:
+ raw = string.Template(open(template).read())
+ targetFile.write(raw.substitute(**mapping))
+
+
+ # write setpermissions helper script
+ template = os.path.join(templateFolder, 'setpermissions.sh.template')
+ target = os.path.join(os.getcwd(), 'docker/setpermissions.sh')
+ substituteAndWrite(template, target, {})
+ print("--> Created permission helper script for easier container setup.")
+
+ # write welcome message file
+ template = os.path.join(templateFolder, 'WELCOME.template')
+ target = os.path.join(os.getcwd(), 'docker/WELCOME')
+ substituteAndWrite(template, target,
+ {'modName': moduleName, 'modFolder': moduleName})
+ print("--> Created welcome message displayed on Docker container startup.")
+
+ # write readme file
+ template = os.path.join(templateFolder, 'README.md.template')
+ target = os.path.join(os.getcwd(), 'docker/README.md')
+ substituteAndWrite(template, target,
+ {'modName': moduleName, 'dockerTag': dockerTag})
+ print("--> Created README.md on how to use the docker image.")
+
+ # write helper file for container spin-up (make it executable after creation)
+ template = os.path.join(templateFolder, 'docker.sh.template')
+ target = os.path.join(os.getcwd(), 'docker/docker_{}.sh'.format(dockerTag))
+ substituteAndWrite(template, target, {'dockerTag': dockerTag})
+ os.system("chmod +x " + target)
+ print("--> Created helper script to spin up the docker container.")
+
+ # write the docker file
+ template = os.path.join(templateFolder, 'Dockerfile.template')
+ target = os.path.join(os.getcwd(), 'docker/Dockerfile')
+ substituteAndWrite(template, target,
+ {
+ 'modName': moduleName,
+ 'modMaintainer': moduleMaintainer,
+ 'dockerTag': dockerTag,
+ 'instScript': installScriptName
+ })
+ print("--> Created Dockerfile. You can adapt it to your needs.")
+ print()
+ print("Do you want to directly build the Docker image? [y/N]")
+
+ build = input()
+ if build == "y" or build == "Y":
+ print("Building Docker image... this may take several minutes.")
+ try:
+ os.chdir('docker')
+ subprocess.run(['docker', 'build',
+ '-f', 'Dockerfile',
+ '-t', dockerTag, '.'], check=True)
+ os.chdir('../')
+ except Exception:
+ os.chdir('../')
+ sys.exit("ERROR: docker image build failed")
+
+ print()
+ print("Successfully built image: {}. "
+ "Have a look at docker/README.md.".format(dockerTag))
+ print("Check the container by running "
+ "'docker run -it {} /bin/bash' in the same".format(dockerTag))
+ print("directory as the Dockerfile, and try using the convenience script "
+ "docker_{}.sh".format(dockerTag))
+ print("See docker/README.md for more information.")
+ else:
+ print("You can build your Docker image later by running "
+ "'docker build -f Dockerfile -t {}'".format(dockerTag))
+ print("from within the folder 'docker' that was created by this script, "
+ "and in which you should find the 'Dockerfile'.")
diff --git a/bin/extract_as_new_module.py b/bin/extract_as_new_module.py
new file mode 100755
index 0000000000000000000000000000000000000000..cca14fc2b8694d9ee9e8fc4416915d5ea5befc8c
--- /dev/null
+++ b/bin/extract_as_new_module.py
@@ -0,0 +1,706 @@
+#!/usr/bin/env python3
+"""
+This script extracts some specified applications into a separate Dune module.
+For example make a dumux-pub repository accompanying a scientific paper.
+"""
+
+import os
+import os.path
+import subprocess
+import argparse
+import shutil
+import textwrap
+import itertools
+import multiprocessing as mp
+from distutils.dir_util import copy_tree
+from pathlib import Path
+from functools import partial
+
+from util.moduleinfo import getDependencies
+from util.common import (
+ callFromPath,
+ runCommand,
+ userQuery,
+ queryYesNo,
+ includedCppProjectHeaders,
+ findMatchingFiles,
+ versionTable,
+)
+from util.installscript import (
+ makeInstallScript,
+ supportedLanguages,
+ getScriptExtension,
+ filterDependencies,
+ addDependencyVersions,
+ addDependencyPatches,
+ makeScriptWriter,
+)
+
+
+def readmeFileName():
+ """The default readme filename"""
+ return "README.md"
+
+
+def makeStringList(items, indentation=" "):
+ """Make a markdown list given the items"""
+ return "\n".join([(indentation + "- " + str(it)) for it in items])
+
+
+def replaceFileContent(file, newContent):
+ """Replace the files content with the new content"""
+ with open(file, "w") as newFile:
+ newFile.write(newContent)
+
+
+def appendFileContent(file, content):
+ """Append some content to the given file"""
+ with open(file, "a") as newFile:
+ newFile.write(content)
+
+
+def isInSubTree(file, base):
+ """Check if a file is in the folder tree starting from base"""
+ return Path(base).resolve() in Path(file).resolve().parents
+
+
+def removeRedundantFolders(folders):
+ """Remove folders that are duplicates or that are contained in a parent folder"""
+ uniqueFolders = list(set(folders))
+ return [
+ sf
+ for sf in uniqueFolders
+ if not any(isInSubTree(sf, base) for base in uniqueFolders)
+ ]
+
+
+def checkModuleFolder(moduleDirectory):
+ """Verify a module folder (module to be extracted from)"""
+ if not os.path.isdir(moduleDirectory):
+ raise Exception(
+ f"Module folder {moduleDirectory} not found. "
+ f"Make sure to run this script from one level above {moduleDirectory}."
+ )
+
+
+def checkSubFolders(moduleDirectory, subDirectories):
+ """Verify that subfolders exist in the module"""
+ for subDirectory in subDirectories:
+ path = Path(moduleDirectory) / Path(subDirectory)
+ errMsg = "Cannot handle the given folder {}".format(str(path))
+ if not path.exists():
+ raise Exception(errMsg + " because it does not exist.")
+ if not path.is_dir():
+ raise Exception(errMsg + " because it is not a directory.")
+
+
+def extractSourceFiles(moduleDirectory, subFolder):
+ """Find all applications (*.cc files) in the given subfolder"""
+ sources = []
+ for folder in subFolder:
+ folderPath = os.path.abspath(os.path.join(moduleDirectory, folder))
+ curSources = findMatchingFiles(folderPath, "*.cc")
+ sources += [os.path.normpath(os.path.join(folderPath, s)) for s in curSources]
+
+ if not sources:
+ raise Exception(
+ "No sources found in the provided subFolder.",
+ f" Run '{os.path.abspath(__file__)} --help' for details.",
+ )
+
+ return sources
+
+
+def runDuneProject():
+ """Run the duneproject script"""
+ duneProjectCommand = shutil.which("duneproject", path="dune-common/bin")
+ if duneProjectCommand is None:
+ raise IOError("Could not find 'duneproject' in dune-common/bin.")
+ try:
+ subprocess.run([duneProjectCommand], check=True)
+ except Exception as exc:
+ raise Exception("Error during creation of the new module.") from exc
+
+
+def detectNewModule():
+ """
+ Detect a new module created by dunecontrol (using the timestamp)
+ This has to be used right after runDuneProject.
+ """
+
+ print("\nDetecting the newly created module")
+ newModule = max([d for d in os.listdir() if os.path.isdir(d)], key=os.path.getmtime)
+
+ isCorrectModule = queryYesNo(
+ f"Found '{newModule}' to be the new module. Is this correct?", default=None
+ )
+ if not isCorrectModule:
+ newModule = userQuery("Please provide the name of the new module:")
+
+ duneModuleFile = f"{newModule}/dune.module"
+ if not os.path.exists(duneModuleFile):
+ print(f"Could not find module file {duneModuleFile}")
+ return newModule
+
+
+def copySubFolders(subFolder, oldPath, newPath):
+ """Copy folders from old path to new path"""
+ for b in subFolder:
+ copy_tree(os.path.join(oldPath, b), os.path.join(newPath, b))
+
+
+def addFoldersToCMakeLists(modulePath, subFolder):
+ """Make sure CMakeLists.txt files exist and contain the necessary add_subdirectory calls"""
+ for subFolderPATH in subFolder:
+ nestedLevel = subFolderPATH.count(os.sep) + 1
+ cmakeListsFile = os.path.join(modulePath, "CMakeLists.txt")
+ for i in range(nestedLevel):
+ if os.path.exists(cmakeListsFile):
+ with open(cmakeListsFile, "r") as cml:
+ lines = list(reversed(cml.readlines()))
+
+ idx = -1
+ for line in lines:
+ if line.startswith("add_subdirectory"):
+ idx = lines.index(line)
+ break
+
+ newLines = lines[0:idx]
+ newLines += [
+ f"add_subdirectory({subFolderPATH.split(os.path.sep)[i]})\n"
+ ]
+ newLines += lines[idx:]
+ newContent = "".join(line for line in reversed(newLines))
+
+ replaceFileContent(cmakeListsFile, newContent)
+ else:
+ with open(cmakeListsFile, "w") as cml:
+ cml.write(
+ f"add_subdirectory({subFolderPATH.split(os.path.sep)[i]})\n"
+ )
+ cmakeListsFile = os.path.join(
+ os.path.dirname(cmakeListsFile), subFolderPATH.split(os.path.sep)[i], "CMakeLists.txt"
+ )
+
+
+def findHeaders(modulePath, sourceFiles):
+ """Find header included (recursively) in the given source files"""
+ with mp.Pool() as p:
+ headers = itertools.chain.from_iterable(
+ p.map(
+ partial(includedCppProjectHeaders, projectBase=modulePath), sourceFiles
+ )
+ )
+ return list(set(headers))
+
+
+def copyFiles(filePaths, oldModulePath, newModulePath):
+ """Copy files from the old to the new module"""
+ newFiles = []
+ for filePath in filePaths:
+ newDirectory = os.path.join(
+ newModulePath, os.path.relpath(os.path.dirname(filePath), oldModulePath)
+ )
+ newFile = os.path.join(newModulePath, os.path.relpath(filePath, oldModulePath))
+
+ newFiles.append(newFile)
+ os.makedirs(newDirectory, exist_ok=True)
+ shutil.copy(filePath, newFile)
+ return newFiles
+
+
+def foldersWithoutSourceFiles(modulePath, checkSubFolder, sources):
+ """Find those folders that do not contain a *.cc source file (application)"""
+ sourceDirectories = [os.path.dirname(s) for s in sources]
+ sourceDirectories = list(set(sourceDirectories))
+
+ def hasChildSourceDirectory(directory):
+ return any(isInSubTree(s, directory) for s in sourceDirectories)
+
+ def isNotASourceDirectory(directory):
+ return directory not in sourceDirectories and not hasChildSourceDirectory(
+ directory
+ )
+
+ noSourceDirectories = []
+ for sf in checkSubFolder:
+ for root, dirs, _ in os.walk(os.path.join(modulePath, sf)):
+ for directory in dirs:
+ directory = os.path.join(root, directory)
+ if isNotASourceDirectory(directory):
+ noSourceDirectories.append(directory)
+ noSourceDirectories = removeRedundantFolders(noSourceDirectories)
+
+ def removeEmptyParents():
+ folderMap = {}
+ for f in noSourceDirectories:
+ parent = os.path.dirname(f)
+ if parent not in folderMap:
+ folderMap[parent] = []
+ folderMap[parent].append(f)
+
+ for parent, folders in folderMap.items():
+ found = set(folders)
+ for root, dirs, _ in os.walk(parent):
+ dirs = [os.path.join(root, d) for d in dirs]
+ if set(dirs) == found and isNotASourceDirectory(parent):
+ for entry in found:
+ noSourceDirectories.remove(entry)
+ noSourceDirectories.append(parent)
+ return noSourceDirectories
+
+ return removeEmptyParents()
+
+
+def removeFolder(modulePath, subFolder):
+ """Remove a folder from the newly created module (adjust CMakeLists.txt)"""
+
+ def removeAddSubdirectoryCommand(cmakeLists, folder):
+ with open(cmakeLists, "r") as cml:
+ content = cml.read()
+
+ key = f"add_subdirectory({folder})\n"
+ if key in content:
+ replaceFileContent(cmakeLists, content.replace(key, ""))
+ return True
+ return False
+
+ subFolderPath = os.path.abspath(os.path.join(modulePath, subFolder))
+ subFolderName = os.path.basename(subFolderPath.rstrip(os.sep))
+
+ mainCML = os.path.join(modulePath, "CMakeLists.txt")
+ parentCML = os.path.join(subFolderPath, "../CMakeLists.txt")
+ if not removeAddSubdirectoryCommand(mainCML, subFolder):
+ if os.path.exists(parentCML):
+ removeAddSubdirectoryCommand(parentCML, subFolderName)
+ shutil.rmtree(subFolderPath)
+
+
+def guideFolderDeletion(modulePath, candidates):
+ """Interactive process to delete folders that may not be needed"""
+ candidateList = makeStringList(candidates)
+ print(
+ "\n"
+ "Could not automatically determine if the following directories\n"
+ "contain data that is essential for the extracted applications:\n"
+ "\n"
+ f"{candidateList}\n"
+ )
+
+ deleted = []
+ if queryYesNo(
+ "Do you want to remove some of them "
+ "(by choosing 'no' they are all preserved)?",
+ default="no",
+ ):
+ for folder in foldersWithoutSources:
+ if queryYesNo(f"Do you want to delete the folder {folder}?", default="yes"):
+ relativePath = os.path.relpath(folder, modulePath)
+ removeFolder(newModulePath, relativePath)
+ deleted.append(relativePath)
+ return deleted
+
+
+def pubProjectURL(projectName):
+ """Default URL for dumux-pub modules"""
+ baseURL = "https://git.iws.uni-stuttgart.de/dumux-pub/"
+ return baseURL + "{}.git".format(projectName.lower())
+
+
+def queryEmptyRemoteURL():
+ """Interactive process to determine the remote URL of the new repository"""
+ while True:
+ if queryYesNo("Is your repository hosted in dumux-pub?"):
+ project = input(
+ "Please provide the name of your project "
+ "(usually AuthorLastNameYearX, e.g. Luigi2020b)\n"
+ )
+ remote = pubProjectURL(project)
+ else:
+ remote = input("Please provide the URL of your repository:\n")
+
+ try:
+ print("Checking the repo (you may have to introduce credentials):")
+ remoteContent = runCommand(
+ "git ls-remote {}".format(remote), suppressTraceBack=True
+ )
+ except subprocess.CalledProcessError:
+ print(" - Could not find your repo at {}. ".format(remote))
+ print(" - Please revisit the provided information.")
+ continue
+
+ if remoteContent == "":
+ return remote
+
+ print(
+ "- Remote repository is not empty. Cannot push to non-empty repository.\n"
+ "- Please provide a URL of an empty/bare Git repository."
+ )
+
+
+def runGitCommand(path, cmd):
+ """Helper function to call and log git commands"""
+ print(f"Running {cmd} (you may have to provide your credentials)")
+ callFromPath(path)(runCommand)(cmd)
+
+
+def pushRepository(modulePath, remoteURL, defaultBranch="master"):
+ """Push to the main branch of the new repository"""
+ runGitCommand(modulePath, f"git push -u {remoteURL} {defaultBranch}")
+
+
+def guideRepositoryInitialization(modulePath):
+ """Initialize new module as git repo (and push if remote is provided)"""
+ hasRepo = queryYesNo(
+ "Do you have an empty remote repository to push the code to?"
+ " (If not, we recommend that you create one and answer with 'yes')"
+ )
+ remoteURL = None if not hasRepo else queryEmptyRemoteURL()
+
+ runGitCommand(modulePath, "git init")
+ runGitCommand(modulePath, "git add .")
+ runGitCommand(modulePath, 'git commit -m "Initial commit"')
+
+ if hasRepo:
+ runGitCommand(modulePath, "git remote add origin {}".format(remoteURL))
+ pushRepository(modulePath, remoteURL)
+
+ return remoteURL
+
+
+def dependenciesAndPatches(modulePath, skip=[]):
+ """Determine the module's dependencies"""
+ try:
+ print(
+ f"Determining dependencies of the module: {modulePath}..."
+ " this may take several minutes"
+ )
+ deps = getDependencies(modulePath)
+ deps = filterDependencies(deps, skip)
+ deps = addDependencyVersions(deps, ignoreUntracked=True)
+ deps = addDependencyPatches(deps)
+ except Exception as exc:
+ raise Exception("Error getting the dependencies.") from exc
+ return deps
+
+
+def guideVersionsReadme(modulePath, dependencies, readme, remoteURL=None):
+ """Write detailed version information to the new readme file"""
+ writeVersionInfo = queryYesNo(
+ "Write detailed version information"
+ f" (folder/branch/commits/dates) into {readmeFileName()}?"
+ )
+ if writeVersionInfo:
+ table = versionTable(dependencies)
+ appendFileContent(readme, f"\n## Version Information\n\n{table}\n")
+ runGitCommand(
+ modulePath, f'git commit {readme} -m "[readme] Update version information"'
+ )
+
+ if remoteURL:
+ pushRepository(modulePath, remoteURL)
+
+
+def guideInstallScriptGeneration(modulePath, dependencies, scriptNameBody):
+ """Interactively add an install script (or not)"""
+ language = userQuery(
+ "In which language would you like to generate the install script?"
+ " (choose 'none' if you don't want an installation script)",
+ supportedLanguages() + ["none"],
+ )
+
+ if language == "none":
+ return None
+
+ installScriptName = scriptNameBody + getScriptExtension(language)
+
+ try:
+ makeInstallScript(
+ modPath=modulePath,
+ dependencies=dependencies,
+ scriptName=installScriptName,
+ writer=makeScriptWriter(language),
+ topFolderName="",
+ )
+ except Exception as e:
+ print(f"Error during install script generation: {e}")
+
+ return installScriptName
+
+
+def processInstallScript(script, modulePath, readme, remoteURL=None):
+ """Add install script and installation instructions to the new module"""
+ newScript = os.path.join(modulePath, script)
+ shutil.move(script, newScript)
+ subprocess.call(["chmod", "u+x", newScript])
+ runGitCommand(modulePath, f"git add {script}")
+ runGitCommand(modulePath, 'git commit -m "Add installation script"')
+
+ appendFileContent(readme, infoReadmeInstallation(remoteURL, script, newModuleName))
+ runGitCommand(
+ modulePath,
+ f'git commit {readme} -m "[readme] Update installation instructions"',
+ )
+
+ if remoteURL:
+ pushRepository(modulePath, remoteURL)
+ else:
+ print(
+ "\n"
+ "Please remember to manually fix the installation instructions\n"
+ "once you know the remote URL where your repository will be hosted"
+ )
+
+
+def noRemoteURLInfo(newModule):
+ """Print information if not remote is present"""
+ return textwrap.dedent(
+ f"""\
+ No remote URL given for new module {newModule}.
+ Please remember to replace the placeholder `$remoteurl`
+ in README.md of the new module {newModule} once a remote is available.
+ """
+ )
+
+
+def infoInitial(moduleDirectory, subFolder, sourceFiles):
+ """Some general information for users of the script"""
+
+ sourcesList = makeStringList(sourceFiles)
+ subFolderList = makeStringList(subFolder)
+
+ return textwrap.dedent(
+ """\
+ This script will extract the following subfolder(s) of
+ the module '{0}':
+ {1}
+
+ and all headers contained in '{0}'
+ that are required to build the exectutables from the sources:
+ {2}
+
+ The extracted files are copied into a new DUNE module retaining the directory
+ structure. The files required for creating a working DUNE module (such as
+ CMakeLists.txt) will be created and/or updated.
+
+ In the next step, the 'duneproject' script will be executed to guide the
+ creation of your new DUNE module. Please answer all upcoming queries to the
+ best of your knowledge.
+
+ Important: the new module should NOT depend on the module '{0}'
+ """
+ ).format(moduleDirectory, subFolderList, sourcesList)
+
+
+def infoReadmeMain(moduleDirectory, subFolder, sourceFiles):
+ """Main part of the README.md document"""
+
+ def relativePath(p):
+ return os.path.relpath(p, moduleDirectory)
+
+ subFolderString = "".join([f"* `{d}`\n" for d in subFolder])
+ sourceString = "".join([f"* `{relativePath(s)}`\n" for s in sourceFiles])
+
+ return textwrap.dedent(
+ """\
+ This file has been created automatically. Please adapt it to your needs.
+
+ ## Content
+
+ The content of this DUNE module was extracted from the module `{0}`.
+ In particular, the following subFolder of `{0}` have been extracted:
+ {1}
+
+ Additionally, all headers in `{0}` that are required to build the
+ executables from the sources
+ {2}
+
+ have been extracted. You can configure the module just like any other DUNE
+ module by using `dunecontrol`. For building and running the executables,
+ please go to the build folders corresponding to the sources listed above.
+
+ """
+ ).format(moduleDirectory, subFolderString, sourceString)
+
+
+def infoReadmeInstallation(remoteURL, installScriptName, newModuleName):
+ """Installation part of the README.md document"""
+
+ installScriptPath = os.path.join(newModuleName, installScriptName)
+ remoteHints = ""
+ if not remoteURL:
+ remoteURL = "$remoteurl"
+ remoteHints = "\nImportant: $remoteurl has to be corrected!\n"
+ return textwrap.dedent(
+ """\
+
+ ## Installation
+
+ The installation procedure is done as follows:
+ Create a root folder, e.g. `DUMUX`, enter the previously created folder,
+ clone this repository and use the install script `{0}`
+ provided in this repository to install all dependent modules.
+ {1}
+ ```sh
+ mkdir DUMUX
+ cd DUMUX
+ git clone {2} {3}
+ ./{4}
+ ```
+
+ This will clone all modules into the directory `DUMUX`,
+ configure your module with `dunecontrol` and build tests.
+
+ """
+ ).format(installScriptName, remoteHints, remoteURL, newModuleName, installScriptPath)
+
+
+def infoFinal(newModuleName):
+ """Print success message with information"""
+ return textwrap.dedent(
+ f"""\
+ ========================================================================
+
+ The module was extracted successfully!
+
+ The extracted module is contained in the subfolder '{newModuleName}'.
+ You can configure it with
+ ./dune-common/bin/dunecontrol --opts={newModuleName}/cmake.opts --only={newModuleName} all
+ """
+ )
+
+
+###############
+# Main script #
+if __name__ == "__main__":
+
+ # set script parameters
+ EPILOG = """
+ -----------------------------------------------------------
+ The script has to be called one level above moduleDirectory.
+ At least one of the subFolder (FOLDER_1 [FOLDER_2 ...]) has
+ to contain a source file *.cc of an executable for which
+ you would like to timber a table in dumux-pub.)
+ -----------------------------------------------------------
+
+ Example usage:
+ ./dumux/bin/extractmodule/extract_as_new_module.py dumux-fracture appl test
+
+ (extracts the subFolder appl and test from the module dumux-fracture)
+
+ """
+
+ parser = argparse.ArgumentParser(
+ prog="extract_as_new_module.py",
+ usage="./dumux/bin/extractmodule/extract_as_new_module.py"
+ " module_dir SUBFOLDER_1 [SUBFOLDER_2 ...]",
+ description="This script extracts subFolder of a given DUNE module"
+ " into a new DUNE module.",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=EPILOG,
+ )
+ parser.add_argument(
+ "module_dir", help="Module from which the subfolder is extracted"
+ )
+ parser.add_argument(
+ "subfolder", nargs="+", help='subfolder(s) of "module_dir" to be extracted'
+ )
+
+ # prepare input
+ args = vars(parser.parse_args())
+ moduleDirectory = args["module_dir"].strip(os.sep)
+ modulePath = os.path.abspath(moduleDirectory)
+ baseFolder = os.path.abspath(os.path.join(moduleDirectory, "../"))
+ subFolder = removeRedundantFolders(args["subfolder"])
+
+ # make sure module directory is valid
+ checkModuleFolder(moduleDirectory)
+ # make sure subdirectories exist
+ checkSubFolders(moduleDirectory, subFolder)
+
+ # find main source files (i.e. applications) to extract
+ sourceFiles = extractSourceFiles(moduleDirectory, subFolder)
+
+ # guide user through new module creation
+ print(infoInitial(moduleDirectory, subFolder, sourceFiles))
+ input(
+ "Please read the above carefully and press [Enter] to proceed or abort with [Ctrl-C]..."
+ )
+
+ # duneproject creates a new Dune module
+ runDuneProject()
+
+ # find the new module (newest folder) created by duneproject
+ # we could also parse the output of duneproject but this solution
+ # seems easier to code
+ newModuleName = detectNewModule()
+ newModulePath = os.path.join(baseFolder, newModuleName)
+
+ # prepare all data in new module
+ copySubFolders(subFolder, modulePath, newModulePath)
+ addFoldersToCMakeLists(newModulePath, subFolder)
+
+ # find all headers necessary to build the applications
+ headers = findHeaders(modulePath, sourceFiles)
+ newHeaders = copyFiles(headers, modulePath, newModulePath)
+ newSourceFiles = copyFiles(sourceFiles, modulePath, newModulePath)
+
+ # copy the gitignore file from dumux
+ copyFiles(
+ [os.path.join(baseFolder, "dumux/.gitignore")],
+ os.path.join(baseFolder, "dumux"),
+ newModulePath,
+ )
+
+ # copy the cmake.opts file from dumux
+ copyFiles(
+ [os.path.join(baseFolder, "dumux/cmake.opts")],
+ os.path.join(baseFolder, "dumux"),
+ newModulePath,
+ )
+
+ # guide user through removal of possibly unnecessary folders
+ foldersWithoutSources = foldersWithoutSourceFiles(
+ newModulePath, subFolder, newHeaders + newSourceFiles
+ )
+
+ actualSubFolder = subFolder
+ if foldersWithoutSources:
+ deletedFolders = guideFolderDeletion(newModulePath, foldersWithoutSources)
+ actualSubFolder = [s for s in subFolder if s not in deletedFolders]
+
+ # remove stuff that is created when running duneproject
+ if os.path.join(newModulePath, "dune") not in foldersWithoutSources:
+ removeFolder(newModulePath, "dune")
+ if os.path.join(newModulePath, "src") not in foldersWithoutSources:
+ removeFolder(newModulePath, "src")
+
+ # delete auto-generated README file
+ duneReadme = os.path.join(newModulePath, "README")
+ if os.path.exists(duneReadme):
+ os.remove(duneReadme)
+
+ # prepare new README.md file
+ newReadme = os.path.join(newModulePath, readmeFileName())
+ replaceFileContent(
+ newReadme, infoReadmeMain(moduleDirectory, actualSubFolder, sourceFiles)
+ )
+
+ # try to initialize repo (to use its URL in later steps)
+ remoteURL = guideRepositoryInitialization(newModulePath)
+ if not remoteURL:
+ print(noRemoteURLInfo(newModuleName))
+
+ # make install script & finalize readme
+ deps = dependenciesAndPatches(newModulePath, [modulePath])
+ if deps:
+ guideVersionsReadme(newModulePath, deps, newReadme, remoteURL)
+
+ installScript = "install_" + newModuleName
+ installScript = guideInstallScriptGeneration(newModulePath, deps, installScript)
+ if installScript is not None:
+ processInstallScript(installScript, newModulePath, newReadme, remoteURL)
+ else:
+ print("No dependencies found. Skipping install script generation")
+
+ print(infoFinal(newModuleName))
diff --git a/bin/make_installscript.py b/bin/make_installscript.py
new file mode 100755
index 0000000000000000000000000000000000000000..12103d310100d5be9972dc59f02b3b77274b1ce2
--- /dev/null
+++ b/bin/make_installscript.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+""""
+Script to generate an install script for a dune-module,
+accounting for non-published commits and local changes
+"""
+
+import sys
+import os
+import argparse
+import subprocess
+
+from util.moduleinfo import getDependencies, getModuleInfo
+from util.installscript import (
+ addDependencyPatches,
+ addDependencyVersions,
+ getDefaultScriptName,
+ filterDependencies,
+ makeInstallScript,
+ makeScriptWriter,
+ printProgressInfo,
+ printFoundDependencies,
+ printFoundVersionInfo,
+ printFinalMessage,
+ supportedLanguages,
+)
+
+
+if __name__ == "__main__":
+
+ ###################
+ # parse arguments
+ parser = argparse.ArgumentParser(
+ description="This script generates an install script for your module, "
+ "taking into account non-published commits & changes.\n"
+ "This expects that all modules are git repositories and "
+ "have a remote origin URL defined."
+ )
+ parser.add_argument(
+ "-p", "--path", required=True, help="The path to your dune module"
+ )
+ parser.add_argument(
+ "-f",
+ "--filename",
+ required=False,
+ help="File in which to write the install script",
+ )
+ parser.add_argument(
+ "-i",
+ "--ignoreuntracked",
+ required=False,
+ action="store_true",
+ help="Use this to ignore untracked files present",
+ )
+ parser.add_argument(
+ "-t",
+ "--topfoldername",
+ required=False,
+ default="DUMUX",
+ help="Folder that the install script creates upon"
+ "execution to install the modules in. If you "
+ "pass an empty string, no folder will be created "
+ "and installation happens in place.",
+ )
+ parser.add_argument(
+ "-o",
+ "--optsfile",
+ required=False,
+ help="Provide custom opts file to be used for project "
+ "configuration. Note that this file is required "
+ "to be contained and committed in the module or "
+ "one of its dependencies.",
+ )
+ parser.add_argument(
+ "-s",
+ "--skipfolders",
+ required=False,
+ nargs="+",
+ help="a list of module folders to be skipped",
+ )
+ parser.add_argument(
+ "-l",
+ "--language",
+ required=False,
+ default="python",
+ choices=supportedLanguages(),
+ help="Language in which to write the install script",
+ )
+
+ cmdArgs = vars(parser.parse_args())
+
+ modPath = cmdArgs["path"]
+ skipFolders = cmdArgs["skipfolders"]
+ if skipFolders:
+ skipFolders = list(set(skipFolders))
+
+ printProgressInfo(["Determining the module dependencies"])
+ deps = getDependencies(modPath, verbose=True, includeSelf=True)
+ deps = filterDependencies(deps, skipFolders)
+ printFoundDependencies(deps)
+ if not deps:
+ sys.exit("No dependencies found. Exiting.")
+
+ printProgressInfo(["Determining the module versions"])
+ deps = addDependencyVersions(deps, cmdArgs.get("ignoreuntracked", False))
+ printFoundVersionInfo(deps)
+
+ printProgressInfo(["Making patches for unpublished & uncommited changes"])
+ deps = addDependencyPatches(deps)
+
+ # actual script generation
+ modPath = os.path.abspath(modPath)
+ modName = getModuleInfo(modPath, "Module")
+ printProgressInfo(
+ [
+ "Creating install script for module '{}' in folder '{}'".format(
+ modName, modPath
+ )
+ ]
+ )
+
+ language = cmdArgs["language"]
+ scriptName = cmdArgs.get("filename", None)
+ if not scriptName:
+ scriptName = getDefaultScriptName(modName, language)
+
+ makeInstallScript(
+ modPath=modPath,
+ dependencies=deps,
+ scriptName=scriptName,
+ writer=makeScriptWriter(language),
+ topFolderName=cmdArgs.get("topfoldername", None),
+ optsFile=cmdArgs.get("optsFile", None),
+ )
+
+ subprocess.call(["chmod", "u+x", scriptName])
+ printProgressInfo([f"Successfully created install script '{scriptName}'"])
+ printFinalMessage(scriptName, cmdArgs.get("topfoldername", None))
diff --git a/bin/util/remove_clutter_after_last_endif.py b/bin/remove_clutter_after_last_endif.py
similarity index 100%
rename from bin/util/remove_clutter_after_last_endif.py
rename to bin/remove_clutter_after_last_endif.py
diff --git a/bin/util/__init__.py b/bin/util/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/bin/util/common.py b/bin/util/common.py
index 70296eba3b87a21baecbb1f0e56fab3af711301d..70a1fe378618c792cb5f8476de90dbf0e3e13bec 100644
--- a/bin/util/common.py
+++ b/bin/util/common.py
@@ -1,8 +1,77 @@
import os
+import re
import sys
+import fnmatch
import functools
import subprocess
import traceback
+import datetime
+import shlex
+
+
+TERM_FORMATTING = {
+ "warning": "\033[1;31m",
+ "highlight": "\033[1;34m",
+ "reset": "\033[0m",
+ "none": "",
+}
+
+
+def styledBotPrint(s, style="none", **kwargs):
+ sys.stdout.write("\n🤖 ")
+ sys.stdout.write(TERM_FORMATTING[style])
+ print(s, **kwargs)
+ sys.stdout.write(TERM_FORMATTING["reset"])
+
+
+def addPrefix(prefix, text, separator=' '):
+ return prefix + separator + text
+
+
+def addPrefixToLines(prefix, text, separator=' '):
+ return '\n'.join(
+ addPrefix(prefix, line, separator) for line in text.split('\n')
+ )
+
+
+def escapeCharacter(text, character, escCharacter="\\"):
+ return text.replace(character, f'{escCharacter}{character}')
+
+
+def escapeCharacters(text, characters, escCharacter="\\"):
+ for char in characters:
+ text = escapeCharacter(text, char, escCharacter)
+ return text
+
+
+def indent(text, indentation=' '):
+ text = text.split('\n')
+ text = [indentation + line for line in text]
+ return '\n'.join(text)
+
+
+def makeTable(dictList, config=None, padding=2):
+ if config is None:
+ config = {key: key for d in dictList for key in d}
+
+ def getColWidth(row):
+ return max(len(str(r)) for r in row) + padding*2
+
+ def getCol(key):
+ return [config[key]] + [d.get(key, "") for d in dictList]
+
+ widths = {key: getColWidth(getCol(key)) for key in config}
+
+ def makeRow(rowValues):
+ row = "|"
+ for key in config:
+ row += "{}|".format(rowValues.get(key, "").center(widths[key]))
+ return row
+
+ table = [makeRow({key: config[key] for key in config})]
+ table.append('|' + '|'.join('-'*widths[key] for key in config) + '|')
+ table.extend(makeRow(row) for row in dictList)
+ return '\n'.join(table)
def getCommandErrorHints(command):
@@ -13,12 +82,13 @@ def getCommandErrorHints(command):
return None
-# execute a command and retrieve the output
-def runCommand(command, suppressTraceBack=False, errorMessage=''):
+def runCommand(command, check=True, suppressTraceBack=False, errorMessage=''):
+ """execute a command and retrieve the output"""
+
try:
- return subprocess.run(command,
- shell=True, check=True,
- text=True, capture_output=True).stdout
+ return subprocess.run(
+ shlex.split(command), check=check, text=True, capture_output=True
+ ).stdout
except Exception:
eType, eValue, eTraceback = sys.exc_info()
if suppressTraceBack:
@@ -35,8 +105,9 @@ def runCommand(command, suppressTraceBack=False, errorMessage=''):
print(hints)
-# decorator to call function from within the given path
def callFromPath(path):
+ """decorator to call function from within the given path"""
+
def decorator_callFromPath(callFunc):
@functools.wraps(callFunc)
def wrapper_callFromPath(*args, **kwargs):
@@ -47,3 +118,245 @@ def callFromPath(path):
return result
return wrapper_callFromPath
return decorator_callFromPath
+
+
+def userQuery(query, choices=None):
+ """query something from the user"""
+
+ choicesString = ', '.join(str(c) for c in choices) if choices else ''
+ querySuffix = f" (choices: {choicesString})\n" if choices else ' '
+
+ while True:
+ styledBotPrint(f"{query.strip()}{querySuffix}", style="highlight")
+ inp = input()
+
+ if choices and inp not in choices:
+ styledBotPrint(
+ f"Invalid answer: '{inp}'. Choose from {choicesString}.",
+ style="warning"
+ )
+ else:
+ return inp
+
+
+def queryYesNo(question, default="yes"):
+ """query a yes/no answer from the user"""
+
+ affirmative = ["yes", "y", "ye"]
+ negative = ["no", "n"]
+
+ def getChoices():
+ return ", ".join(c for c in affirmative + negative)
+
+ def isAffirmative(choice): return choice in affirmative
+ def isNegative(choice): return choice in negative
+ def isValid(choice): return isAffirmative(choice) or isNegative(choice)
+
+ if default is not None and not isValid(default):
+ raise ValueError("\nInvalid default answer: '{}', choices: '{}'\n"
+ .format(default, getChoices()))
+
+ if default is None:
+ prompt = " [y/n] "
+ else:
+ prompt = " [Y/n] " if isAffirmative(default) else " [y/N] "
+
+ while True:
+ styledBotPrint(f"{question.strip()}{prompt}", style="highlight", end="")
+ choice = input().lower()
+
+ if default is not None and choice == "":
+ return True if isAffirmative(default) else False
+
+ if not isValid(choice):
+ styledBotPrint(
+ f"Invalid answer: '{choice}'. Choose from '{getChoices()}'",
+ style="warning"
+ )
+ else:
+ return True if isAffirmative(choice) else False
+
+
+def cppHeaderFilter():
+ return lambda fileName: fileName == 'config.h'
+
+
+def includedCppProjectHeaders(file,
+ projectBase,
+ headers=[],
+ headerFilter=cppHeaderFilter()):
+ """get all project headers included by a cpp file"""
+
+ filePath = os.path.join(projectBase, file)
+ if not os.path.exists(filePath):
+ raise IOError(f'Cpp file {filePath} does not exist')
+
+ with open(filePath, 'r') as f:
+ content = f.read()
+ headerInBracket = re.findall(r'#include\s+<(.+?)>', content)
+ headerInQuotation = re.findall(r'#include\s+"(.+?)"', content)
+
+ def process(pathInProject):
+ headerPath = os.path.join(projectBase, pathInProject)
+ if os.path.exists(headerPath):
+ if not headerFilter(pathInProject):
+ if headerPath not in headers:
+ headers.append(headerPath)
+ includedCppProjectHeaders(
+ headerPath, projectBase,
+ headers, headerFilter
+ )
+
+ for header in headerInBracket:
+ process(header)
+ for header in headerInQuotation:
+ absHeaderPath = os.path.join(os.path.dirname(file), header)
+ projectPath = os.path.relpath(absHeaderPath, projectBase)
+ process(projectPath)
+ return headers
+
+
+def findMatchingFiles(path, pattern):
+ """find all files below the given folder that match the given pattern"""
+
+ result = []
+ for root, dirs, files in os.walk(path):
+ relativeRootPath = os.path.relpath(root, path)
+ for file in files:
+ if fnmatch.fnmatch(file, pattern):
+ result.append(os.path.join(relativeRootPath, file))
+ return result
+
+
+def isGitRepository(pathToRepo='.'):
+ try:
+ run = callFromPath(pathToRepo)(runCommand)
+ run('git status')
+ return True
+ except Exception:
+ return False
+
+
+def getRemote(pathToRepo='.'):
+ run = callFromPath(pathToRepo)(runCommand)
+ return run('git ls-remote --get-url').strip('\n')
+
+
+def fetchRepo(remote, pathToRepo='.'):
+ run = callFromPath(pathToRepo)(runCommand)
+ run('git fetch {}'.format(remote))
+
+
+def hasUntrackedFiles(pathToRepo='.'):
+ run = callFromPath(pathToRepo)(runCommand)
+ return run('git ls-files --others --exclude-standard') != ''
+
+
+def isPersistentBranch(branchName):
+ if branchName == 'origin/master':
+ return True
+ if branchName.startswith('origin/releases/'):
+ return True
+ return False
+
+
+# get the most recent commit that also exists on remote master/release branch
+# may be used to find a commit we can use as basis for a pub module
+def mostRecentCommonCommitWithRemote(modFolderPath,
+ branchFilter=isPersistentBranch):
+ run = callFromPath(modFolderPath)(runCommand)
+
+ def findBranches(sha):
+ candidates = run('git branch -r --contains {}'.format(sha)).split('\n')
+ candidates = [branch.strip().split(' ->')[0] for branch in candidates]
+ return list(filter(branchFilter, candidates))
+
+ revList = run('git rev-list HEAD').split('\n')
+ for rev in revList:
+ branches = findBranches(rev)
+ if branches:
+ return branches[0], rev
+
+ raise RuntimeError('Could not find suitable ancestor commit'
+ ' on a branch that matches the given filter')
+
+
+# function to extract persistent, remotely available git versions for all
+def getPersistentVersions(modFolderPaths, ignoreUntracked=False):
+
+ result = {}
+ for modFolderPath in modFolderPaths:
+
+ if not isGitRepository(modFolderPath):
+ raise Exception('Folder is not a git repository')
+
+ if hasUntrackedFiles(modFolderPath) and not ignoreUntracked:
+ raise Exception(
+ "Found untracked files in '{}'. "
+ "Please commit, stash, or remove them. Alternatively, if you "
+ "are sure they are not needed set ignoreUntracked=True"
+ .format(modFolderPath)
+ )
+
+ result[modFolderPath] = {}
+ result[modFolderPath]['remote'] = getRemote(modFolderPath)
+
+ # update remote to make sure we find all upstream commits
+ fetchRepo(result[modFolderPath]['remote'], modFolderPath)
+
+ branch, rev = mostRecentCommonCommitWithRemote(modFolderPath)
+ run = callFromPath(modFolderPath)(runCommand)
+
+ result[modFolderPath]['revision'] = rev
+ result[modFolderPath]['date'] = run(
+ 'git log -n 1 --format=%ai {}'.format(rev)
+ ).strip('\n')
+ result[modFolderPath]['author'] = run(
+ 'git log -n 1 --format=%an {}'.format(rev)
+ ).strip('\n')
+
+ # this may return HEAD if we are on some detached HEAD tree
+ result[modFolderPath]['branch'] = branch
+
+ return result
+
+
+def getPatches(persistentVersions):
+ result = {}
+ for path, gitInfo in persistentVersions.items():
+ run = callFromPath(path)(runCommand)
+
+ uncommittedPatch = run('git diff')
+ unpublishedPatch = run(
+ 'git format-patch --stdout {}'.format(gitInfo['revision'])
+ )
+ untrackedPatch = ''
+ untrackedFiles = run('git ls-files --others --exclude-standard')
+ binaryExtension = (
+ '.png', '.gif', '.jpg', '.tiff', '.bmp', '.DS_Store', '.eot', '.otf', '.ttf', '.woff', '.rgb', '.pdf',
+ )
+ if untrackedFiles:
+ for file in untrackedFiles.splitlines():
+ if not str(file).endswith(binaryExtension):
+ untrackedPatch += run('git --no-pager diff /dev/null {}'.format(file), check=False)
+
+ result[path] = {}
+ result[path]['untracked'] = untrackedPatch if untrackedPatch else None
+ result[path]['unpublished'] = unpublishedPatch if unpublishedPatch else None
+ result[path]['uncommitted'] = uncommittedPatch if uncommittedPatch else None
+ return result
+
+
+def versionTable(versions,
+ config={
+ 'name': 'module name',
+ 'branch': 'branch name',
+ 'revision': 'commit sha',
+ 'date': 'commit date'
+ },
+ padding=2):
+ return makeTable(versions, config)
+
+
+def printVersionTable(versions):
+ print(versionTable(versions=versions))
diff --git a/bin/util/create_cmakelists.py b/bin/util/create_cmakelists.py
deleted file mode 100755
index 63092955909a0558d71614bf82ac4ac1e4c2f096..0000000000000000000000000000000000000000
--- a/bin/util/create_cmakelists.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Create files CMakeLists.txt for the given folder and all subfolders,
-including the add_subdirectory(...) and install(...) commands.
-Defaults to the folder `dumux` that contains the header files,
-if no folder was specified.
-"""
-
-import os
-import argparse
-
-parser = argparse.ArgumentParser()
-parser.add_argument('folder', type=str, nargs='?', help='the folder to create CMakeLists.txt\'s for', default=None)
-args = vars(parser.parse_args())
-
-# default to the dumux folder (relative path to the location of this script)
-if args['folder'] is None:
- rootDir = os.path.dirname(os.path.abspath(__file__)) + "/../../dumux"
-else:
- rootDir = args['folder']
-
-
-ignore_folders = ["", "io/format/fmt", "io/xml"]
-extensions = [".hh", ".inc"]
-for fullFolderName, subFolders, files in os.walk(rootDir):
- # alphabetically sort
- subFolders = sorted(subFolders)
- files = sorted(files)
- # get folder name relative to dumux
- folderName = fullFolderName.replace(rootDir + '/', '').replace(rootDir, '')
- if folderName not in ignore_folders:
- with open(fullFolderName + "/CMakeLists.txt", "w") as cmakelists:
- # add subfolders
- for subFolder in subFolders:
- cmakelists.write("add_subdirectory({})\n".format(subFolder))
-
- headersExist = False
- for fileName in files:
- ext = os.path.splitext(fileName)[1]
- if ext in extensions:
- headersExist = True
- break
-
- if headersExist:
- if subFolders: cmakelists.write("\n")
- # collect all files to be installed in a CMake variable
- headers_variable = "DUMUX_" + folderName.upper().replace("/", "_") + "_HEADERS"
- cmakelists.write("file(GLOB {}{})\n".format(headers_variable, " *".join([''] + extensions)))
- cmakelists.write("install(FILES ${{{}}}\n".format(headers_variable))
- cmakelists.write(" DESTINATION ${{CMAKE_INSTALL_INCLUDEDIR}}/dumux/{})\n".format(folderName))
diff --git a/bin/util/createdockerimage.py b/bin/util/createdockerimage.py
deleted file mode 100644
index 96619ea5f35c8d5ab09edf7381818c28bb38c786..0000000000000000000000000000000000000000
--- a/bin/util/createdockerimage.py
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-import string
-import shutil
-import argparse
-import subprocess
-from getmoduleinfo import getModuleFile
-from getmoduleinfo import extractModuleInfos
-
-# require python 3
-if sys.version_info[0] < 3:
- sys.exit("\nERROR: Python3 required")
-
-# input argument parser
-parser = argparse.ArgumentParser(
- description="Create a docker image for a given module and install script."
-)
-
-parser.add_argument('-m', '--modulepath',
- required=True,
- help='the path to the your module')
-parser.add_argument('-i', '--installScript',
- required=True,
- help="Specify the installation script")
-parser.add_argument('-t', '--templateFolder',
- required=False,
- help="Specify the folder with the template files")
-
-args = vars(parser.parse_args())
-
-# get information on the module
-modulePath = os.path.abspath(args['modulepath'])
-modInfo = extractModuleInfos(getModuleFile(modulePath),
- ['Module', 'Maintainer'])
-moduleName = modInfo['Module']
-moduleMaintainer = modInfo['Maintainer']
-dockerTag = moduleName.lower() # docker only supports lower case
-
-# get folder with the template files
-templateFolder = args['templateFolder']
-if not templateFolder:
- templateFolder = os.path.join(modulePath, '../dumux/docker')
-if not os.path.exists(templateFolder):
- sys.exit("Template folder {} could not be found".format(templateFolder))
-
-print("*"*54)
-print("\n-- Creating a Docker image for module " + moduleName + " --\n")
-print("*"*54)
-
-if os.path.exists("docker"):
- print("\nA docker folder already exists. "
- "Continue anyway? - will be overwritten - [y/N]\n")
- delete = input()
- if delete == "y" or delete == "Y":
- shutil.rmtree("docker")
- print("--> Deleted old docker folder.")
- else:
- sys.exit("Abort.")
-
-os.mkdir("docker")
-print("--> Created the folder 'docker'.")
-
-# copy install script into docker folder and make it executable
-installScriptPath = args['installScript']
-installScriptName = os.path.split(installScriptPath)[1]
-installScript = os.path.join(os.path.join(os.getcwd(), 'docker'),
- installScriptName)
-shutil.copy(installScriptPath, installScript)
-os.system("chmod +x {}".format(installScript))
-print("--> Using install script: {} to install dependencies for module {}."
- .format(installScript, moduleName))
-
-
-# substitute content from template and write to target
-def substituteAndWrite(template, target, mapping):
- if not os.path.exists(template):
- sys.exit("Template file '" + template + "' could not be found")
- with open(target, 'w') as targetFile:
- raw = string.Template(open(template).read())
- targetFile.write(raw.substitute(**mapping))
-
-
-# write setpermissions helper script
-template = os.path.join(templateFolder, 'setpermissions.sh.template')
-target = os.path.join(os.getcwd(), 'docker/setpermissions.sh')
-substituteAndWrite(template, target, {})
-print("--> Created permission helper script for easier container setup.")
-
-# write welcome message file
-template = os.path.join(templateFolder, 'WELCOME.template')
-target = os.path.join(os.getcwd(), 'docker/WELCOME')
-substituteAndWrite(template, target,
- {'modName': moduleName, 'modFolder': moduleName})
-print("--> Created welcome message displayed on Docker container startup.")
-
-# write readme file
-template = os.path.join(templateFolder, 'README.md.template')
-target = os.path.join(os.getcwd(), 'docker/README.md')
-substituteAndWrite(template, target,
- {'modName': moduleName, 'dockerTag': dockerTag})
-print("--> Created README.md on how to use the docker image.")
-
-# write helper file for container spin-up (make it executable after creation)
-template = os.path.join(templateFolder, 'docker.sh.template')
-target = os.path.join(os.getcwd(), 'docker/docker_{}.sh'.format(dockerTag))
-substituteAndWrite(template, target, {'dockerTag': dockerTag})
-os.system("chmod +x " + target)
-print("--> Created helper script to spin up the docker container.")
-
-# write the docker file
-template = os.path.join(templateFolder, 'Dockerfile.template')
-target = os.path.join(os.getcwd(), 'docker/Dockerfile')
-substituteAndWrite(template, target,
- {
- 'modName': moduleName,
- 'modMaintainer': moduleMaintainer,
- 'dockerTag': dockerTag,
- 'instScript': installScriptName
- })
-print("--> Created Dockerfile. You can adapt it to your needs.")
-print()
-print("Do you want to directly build the Docker image? [y/N]")
-
-build = input()
-if build == "y" or build == "Y":
- print("Building Docker image... this may take several minutes.")
- try:
- os.chdir('docker')
- subprocess.run(['docker', 'build',
- '-f', 'Dockerfile',
- '-t', dockerTag, '.'], check=True)
- os.chdir('../')
- except Exception:
- os.chdir('../')
- sys.exit("ERROR: docker image build failed")
-
- print()
- print("Successfully built image: {}. "
- "Have a look at docker/README.md.".format(dockerTag))
- print("Check the container by running "
- "'docker run -it {} /bin/bash' in the same".format(dockerTag))
- print("directory as the Dockerfile, and try using the convenience script "
- "docker_{}.sh".format(dockerTag))
- print("See docker/README.md for more information.")
-else:
- print("You can build your Docker image later by running "
- "'docker build -f Dockerfile -t {}'".format(dockerTag))
- print("from within the folder 'docker' that was created by this script, "
- "and in which you should find the 'Dockerfile'.")
diff --git a/bin/util/extractmodulepart.py b/bin/util/extractmodulepart.py
deleted file mode 100755
index fbd7cfc69223df929214d97f8f575155a14944a8..0000000000000000000000000000000000000000
--- a/bin/util/extractmodulepart.py
+++ /dev/null
@@ -1,388 +0,0 @@
-#!/usr/bin/env python3
-import sys, glob, os, subprocess
-import os.path
-import argparse
-import shutil
-from distutils.dir_util import copy_tree
-import re
-import threading
-import fnmatch
-from getmoduleinfo import *
-from getusedversions import *
-
-
-"""
-This is a python script for extracting module
-"""
-
-# check if help is needed
-epilog = '''
------------------------------------------------------------
-The script has to be called one level above module_dir.
-At least one of the subfolders (FOLDER_1 [FOLDER_2 ...]) has
-to contain a source file *.cc of an executable for which
-you would like to timber a table in dumux-pub.)
-'''
-parser = argparse.ArgumentParser(prog='extractmodulepart',
- usage= "./extractmodulepart module_dir SUBFOLDER_1 [SUBFOLDER_2 ...]",
- description='This script extracts a subfolder of a DUNE module',
- formatter_class=argparse.RawDescriptionHelpFormatter,
- epilog=epilog)
-parser.add_argument('module_dir', help='Dune module from which the subfolder is extracted')
-parser.add_argument('subfolder', nargs='+', help = 'subfolder(s) of module_dir which you want to extract')
-args = vars(parser.parse_args())
-
-# function to search the header file efficiently with parallel programming
-def search_headers(c_file):
- with open(c_file, 'r') as f:
- content = f.read()
-
- header_in_bracket = re.findall(r'#include\s+<(.+?)>',content)
- header_in_quotation = re.findall(r'#include\s+"(.+?)"',content)
-
- # search for includes relative to the module path
- for header in header_in_bracket + header_in_quotation:
- header_with_path = os.path.join(module_full_path, header)
- if not os.path.exists(header_with_path):
- continue
- if header_with_path not in all_headers:
- all_headers.append(header_with_path)
- thread_ = threading.Thread(target = search_headers, args = (header_with_path, ))
- thread_.start()
-
- # search for includes relative to the path of the including file
- # only allow quoted includes for this
- for header in header_in_quotation:
- if header == "config.h":
- continue
- header_dir_name = os.path.dirname(c_file)
- header_with_path = os.path.join(header_dir_name, header)
- if not os.path.exists(header_with_path):
- continue
- if header_with_path not in all_headers:
- all_headers.append(header_with_path)
- thread_ = threading.Thread(target = search_headers, args = (header_with_path, ))
- thread_.start()
-
-# functions to find the file with specific pattern or name in a direcotry
-def find_files_with_name(name, path):
- result = []
- for root, dirs, files in os.walk(path):
- if name in files:
- result.append(os.path.join(root, name))
- return result
-
-def find_files_with_pattern(pattern, path):
- result = []
- for root, dirs, files in os.walk(path):
- for name in files:
- if fnmatch.fnmatch(name, pattern):
- result.append(os.path.join(root, name))
- return result
-
-# functions to configure CMakeLists.txt files
-def generate_new_content(lines, pattern, replace_str):
- index = lines.find(pattern)
- if index != -1:
- content = lines[0: index] + "\n"
- if replace_str != "":
- content += replace_str + "\n"
- flag = True
- while flag:
- index = lines.find(")", index)
- if index != -1:
- lines = lines[index + 1:]
- else:
- break
-
- index = lines.find(pattern)
- if index != -1:
- content += lines[0: index] + "\n"
- else:
- content += lines + "\n"
- flag = False
- else:
- if replace_str == "":
- content = lines
- else:
- if pattern == "add_subdirectory(":
- content = replace_str + "\n" + lines
- else:
- content = lines + "\n" + replace_str
- return content
-
-def generate_subdirectory_content(dirs):
- content = ""
- for name in dirs:
- content += "add_subdirectory(" + name + ")" + "\n"
- return content
-
-def generate_install_content(header_files, destination):
- if len(header_files) == 0:
- return ""
- content = "install(FILES" + "\n"
- for header_file in header_files:
- content += " " + header_file + "\n"
- content += "DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}" + destination + ")" + "\n"
- return content
-
-def generate_content(cmake_list_txt_file, dirs, header_files, destination):
- subdirectory_content = generate_subdirectory_content(dirs)
- install_content = generate_install_content(header_files, destination)
-
- if os.path.exists(cmake_list_txt_file):
- with open(cmake_list_txt_file, "r", encoding="utf-8") as f:
- content = "".join(f.readlines()).strip()
- f.close()
- content = generate_new_content(content, "add_subdirectory(", subdirectory_content)
- return generate_new_content(content, "install(FILE", install_content)
- else:
- if subdirectory_content == "":
- return install_content
- else:
- return subdirectory_content + "\n" + install_content
-
-def generate_cmake_lists_txt(cmake_list_txt_file, dirs, header_files, destination):
- content = generate_content(cmake_list_txt_file, dirs, header_files, destination)
-
- with open(cmake_list_txt_file, "w", encoding="utf-8") as f:
- if content != "":
- f.write(content)
- f.close()
-
-def check_dir(root_dir):
- if not os.path.exists(root_dir):
- print("root path" + str(root_dir) + "is not exist!")
- return False
- if not os.path.isdir(root_dir):
- print("root path" + str(root_dir) + "is not dir!")
- return False
- return True
-
-def check_str(root_dir):
- if root_dir is None or root_dir.strip() == "":
- return None
- root_dir = root_dir.strip()
- if root_dir.endswith(os.sep):
- root_dir = root_dir[:-1]
- return root_dir
-
-def generate_cmake_lists_txt_file(root_dir):
- root_dir = check_str(root_dir)
- if root_dir is None:
- return
- if not check_dir(root_dir):
- return
- drop_len = len(root_dir)
- for parent, dirs, files in os.walk(root_dir):
- destination = parent[drop_len:]
- header_files = []
- cmake_list_txt_file = os.path.join(parent, "CMakeLists.txt")
- for name in files:
- for suffix in [".h", ".hh"]:
- if name.endswith(suffix):
- header_files.append(name)
- generate_cmake_lists_txt(cmake_list_txt_file, dirs, header_files, destination)
-
-def query_yes_no(question, default="yes"):
- valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
- if default is None:
- prompt = " [y/n] "
- elif default == "yes":
- prompt = " [y/N] "
- elif default == "no":
- prompt = " [Y/n] "
- else:
- raise ValueError("invalid default answer: '%s'" % default)
- while True:
- sys.stdout.write(question + prompt)
- choice = input().lower()
- if default is not None and choice == "":
- return valid[default]
- elif choice in valid:
- return valid[choice]
- else:
- sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n")
-
-if __name__ == "__main__":
-
- module_dir = args['module_dir']
- subfolders = args['subfolder']
-
- # if module_dir contains a slash as last character, delete it
- if module_dir.endswith('/'):
- module_dir = module_dir[:-1]
-
- # check if we are above module_dir
- if not (os.path.isdir(module_dir)):
- print("ERROR: you need to run the script one level above the folder "+module_dir+".")
- print("Run \""+os.path.basename(__file__)+" --help\" for details.")
- exit(1)
-
- # determine all source files in the paths passed as arguments
- script_path = os.getcwd()
- os.chdir(module_dir)
- module_full_path=os.getcwd()
- all_sources=[]
- all_sources_with_path = []
- all_directories=[]
- for dir_path in subfolders:
- if dir_path.startswith(module_dir):
- # the "+1" is required to also remove the "/"
- stripped_path = dir_path[len(module_dir)+1:]
- else:
- stripped_path = dir_path
- full_path = os.path.join(module_full_path, stripped_path)
- directories = " " + stripped_path
- all_directories.append(stripped_path)
- os.chdir(full_path)
- for file in find_files_with_pattern("*.cc", full_path):
- print(file)
- sources = os.path.relpath(file, module_full_path)
- print(sources)
- all_sources.append(sources)
- sourceswithpath = os.path.join(module_full_path, sources)
- all_sources_with_path.append(sourceswithpath)
- os.chdir(module_full_path)
- os.chdir("..") # back to the script folder
-
- # check if sources have been obtained
- if (all_sources == []):
- print("ERROR: no source files *.cc found in the directories "+ " ".join([str(x) for x in subfolders]) +".")
- print("Be sure to provide a list of paths as arguments to this script.")
- print("Run \""+os.path.basename(__file__)+" --help\" for details.")
- exit(1)
-
- # try to find the duneproject script
- dune_project = shutil.which('duneproject', path = "dune-common/bin")
- if (dune_project == None):
- print("ERROR: Could not find duneproject.")
- print("Be sure to either have duneproject in your search path")
- print("or to run this script from a directory that contains duneproject.")
- exit(1)
- else:
- print(dune_project)
-
- # give explanations
- print("\n""This script will\n"
- "- extract the following sub-folders of "+ module_dir +":\n")
- for dir_path in all_directories:
- print(" "+ dir_path + ",")
- print("\n"" and all headers in "+ module_dir + " that are required to build the\n"
- " executables from the sources\n")
- for source in all_sources:
- print(" "+ source +",")
- print("""
-- copy the extracted files into a freshly created DUNE module, retaining the directory structure,
-
-- update/create all required files like CMakeLists.txt, - store the versions of all used Dune module
-
-- and extract their modifications as patches.
-
-Thus, you receive a fully-working DUNE module containing the subset of {0} that is required to run your application.
-duneproject will be run now. The new module should NOT depend on the module in {0}.\n\n""".format(module_dir))
-
- input("Read the above and press [Enter] to proceed...")
-
- # run duneproject
- old_ls = os.listdir()
- subprocess.call([dune_project])
- new_ls = os.listdir()
-
- # determine the new module/directory name
- module_name = (set(new_ls) - set(old_ls)).pop()
- if (module_name == ""):
- print("ERROR: could not find new module. Aborting.")
- exit(1)
- else:
- print()
- print(os.path.basename(__file__) + ": Found new module " + module_name)
- print("Determining required headers...")
- os.chdir(module_name)
- module_path=os.getcwd()
-
- # extract all headers, provide some output and copy everything to the new module
- all_headers = []
- print("The following header files are extracted: ")
- for source in all_sources_with_path:
- search_headers(source)
- for header in all_headers:
- print(header)
- dir_path = os.path.dirname(os.path.realpath(header)).replace(module_dir,module_name,1)
- os.makedirs(dir_path, exist_ok=True)
- shutil.copy(header,dir_path)
- source_dir = os.path.dirname(source)
- source_path = source_dir.replace(module_dir,module_name,1)
- copy_tree(source_dir, source_path)
-
- # delete unnecessary directories to keep the extracted module clean
- shutil.rmtree('dune')
- shutil.rmtree('src')
-
- # set CMakeLists.txt for each directory
- generate_cmake_lists_txt_file(module_path)
- print("The required header files are extracted and CMakeLists are configured.")
- print("=============================================================================")
-
- # create README file
- os.remove("README")
- orig_stdout = sys.stdout
- readme_file = open("README.md", "w+")
-
- readme_dir_str_lists = []
- for dir_path in all_directories:
- readme_dir_str_lists.append("* `" + dir_path + "`,\n")
- readme_source_str_lists = []
- for source in all_sources:
- readme_source_str_lists.append("* `" + source + "`,\n")
- readme_str_args = [module_dir, ''.join(readme_dir_str_lists), ''.join(readme_source_str_lists)]
-
- readme_file.write("""This file has been created automatically. Please adapt it to your needs. \n
-===============================\n
-## Content\n
-"The content of this DUNE module was extracted from the module `{0}`."
-In particular, the following subfolders of `{0}` have been extracted:
-{1}
-Additionally, all headers in `{0}` that are required to build the
-executables from the sources
-{2}
-have been extracted. You can configure the module just like any other DUNE
-module by using `dunecontrol`. For building and running the executables,
-please go to the build folders corresponding to the sources listed above.\n
-""".format(*readme_str_args))
-
- version_script_path = os.path.join(script_path, "dumux/bin/util/getusedversions.py")
- install_script_path = os.path.join(script_path, "dumux/bin/util/makeinstallscript.py")
- os.chdir(script_path)
- if query_yes_no("\nWrite detailed version information (folder/branch/commits/dates) into README.md?"):
- print("Looking for the dune modules in path :" + str(script_path) + "...")
- readme_file.write("===============================\n" + "## Version Information\n")
- versions = getUsedVersions([dep['folder'] for dep in getDependencies(module_full_path)], True)
- print("Writing version information into README.md ...")
- sys.stdout = readme_file
- printVersionTable(versions)
- sys.stdout = orig_stdout
- readme_file.close()
- print("Automatic generation of README.md file is complete.")
- print("=============================================================================")
-
- install_script_name = 'install_' + module_name + '.sh'
- if query_yes_no("\nGenerate install script " + install_script_name + "in your new module " + module_name + "?"):
- os.system("python3 " + install_script_path + " -p" + os.path.join(script_path,module_name) + " -i -s " + module_name)
- shutil.move(install_script_name, os.path.join(module_path, install_script_name))
-
- # output guidence for users
- print("\n"+"*"*80+"\n"+
- """The extracted module is contained in the subfolder \"{0}\".
- You can build it using \"dunecontrol ... —only= {0} all\".
- BEFORE building, you can add the module to dumux-pub by something like:
- (Rename module name if it does not match the AuthorLastNameYearx scheme
- and commit it to the Git repository dumux-pub)
- git clone https://git.iws.uni-stuttgart.de/dumux-pub/AuthorLastNameYearx.git
- sed -i '/Module:/c\Module: AuthorLastNameYearx' {0} /dune.module
- mv {0} /* dumux-pub/AuthorLastNameYearx/.
- cd AuthorLastNameYearx
- git commit -a
- git push""".format(module_name)+
- "\n"+"*"*80)
-
diff --git a/bin/util/extractmodulepart.sh b/bin/util/extractmodulepart.sh
deleted file mode 100755
index 433a79fcd721b96ec76d9cf3d0dd31a3a0b63fdf..0000000000000000000000000000000000000000
--- a/bin/util/extractmodulepart.sh
+++ /dev/null
@@ -1,398 +0,0 @@
-#! /bin/bash
-
-# check if help is needed
-if test "$1" = "--help" || test "$1" = "-help" \
- || test "$1" = "help" || test "$1" = ""; then
- echo ""
- echo "USAGE: $0 MODULE_DIR FOLDER_1 [FOLDER_2 ...]"
- echo ""
- echo "MODULE_DIR is the folder containing the DUNE module from which you"
- echo "want to extract. The script has to be called one level above it."
- echo ""
- echo "The FOLDERs need to indicate subfolders of MODULE_DIR. At least one"
- echo "of them has to contain a source file *.cc of an executable for which"
- echo "you would like to timber a table in dumux-pub."
- exit 0
-fi
-
-MODULE_DIR=$1
-# if MODULE_DIR contains a slash as last character, delete it
-MODULE_DIR=$(echo $MODULE_DIR | sed s'/\/$//')
-# check if we are above MODULE_DIR
-if (! [ -d $MODULE_DIR ]); then
- echo "ERROR: you need to run the script one level above the folder $MODULE_DIR."
- echo "Run \"$0 --help\" for details."
- exit 1
-fi
-
-# determine all source files in the paths passed as arguments
-cd $MODULE_DIR
-MODULE_FULL_PATH=$(pwd)
-ALL_SOURCES=""
-ALL_DIRECTORIES=""
-for DIR_PATH in ${*:2}; do
- STRIPPED_PATH=$(sed 's%.*'"$MODULE_DIR"'/%%g' <(echo "$DIR_PATH"))
- ALL_DIRECTORIES="$ALL_DIRECTORIES $STRIPPED_PATH"
- ALL_SOURCES="$ALL_SOURCES $(find $STRIPPED_PATH -name '*.cc' 2>/dev/null)"
-done
-cd ..
-
-# check if sources have been obtained
-CONTRACTED="$(echo "$ALL_SOURCES" | tr -d " \t\n\r")"
-if test "$CONTRACTED" = ""; then
- echo "ERROR: no source files *.cc found in the directories ${*:2}."
- echo "Be sure to provide a list of paths as arguments to this script."
- echo "Run \"$0 --help\" for details."
- exit 1;
-fi
-
-# try to find the duneproject script
-if hash duneproject 2>/dev/null; then
- DUNE_PROJECT="duneproject"
-else
- DUNE_PROJECT=$(find . -name 'duneproject')
-fi
-if test "$DUNE_PROJECT" = ""; then
- echo "ERROR: Could not find duneproject."
- echo "Be sure to either have duneproject in your search path"
- echo "or to run this script from a directory that contains duneproject."
- exit 1;
-fi
-
-# give explanations
-echo ""
-echo "This script will"
-echo "- extract the following sub-folders of $MODULE_DIR:"
-echo ""
-for DIR_PATH in $ALL_DIRECTORIES; do
- echo " $DIR_PATH,"
-done
-echo ""
-echo " and all headers in $MODULE_DIR that are required to build the"
-echo " executables from the sources"
-echo ""
-for SOURCE in $ALL_SOURCES; do
- echo " $SOURCE,"
-done
-echo ""
-echo "- copy the extracted files into a freshly created DUNE module, retaining the"
-echo " directory structure,"
-echo ""
-echo "- update/create all required files like CMakeLists.txt,"
-echo ""
-echo "- store the versions of all used Dune module"
-echo ""
-echo "- and extract their modifications as patches."
-echo ""
-echo "Thus, you receive a fully-working DUNE module containing the subset of"
-echo "$MODULE_DIR that is required to run your application."
-echo ""
-echo "duneproject will be run now. The new module should NOT depend on the"
-echo "module in $MODULE_DIR."
-echo ""
-read -p "Read the above and press [Enter] to proceed..."
-
-# run duneproject
-OLD_LS="$(ls)"
-$DUNE_PROJECT
-NEW_LS="$(ls)"
-
-# determine the new module/directory name
-DIFF_OUTPUT=$(diff <(echo "$OLD_LS" ) <(echo "$NEW_LS"))
-FOUND="0"
-MODULE_NAME=""
-for WORD in $DIFF_OUTPUT; do
- if test "$FOUND" = "1"; then
- MODULE_NAME=$WORD
- fi
- if test "$WORD" = ">"; then
- FOUND="1"
- fi
-done
-if test "$MODULE_NAME" = ""; then
- echo "ERROR: could not find new module. Aborting."
- exit 1
-else
- echo ""
- echo "$(basename $0): Found new module $MODULE_NAME"
-fi
-
-echo -n "Determining required headers..."
-cd $MODULE_DIR
-
-# initialize lists to hold required headers
-LAST_REQUIRED_HEADERS=""
-REQUIRED_HEADERS="$ALL_SOURCES"
-
-while test "$LAST_REQUIRED_HEADERS" != "$REQUIRED_HEADERS"; do
- echo -n "."
- LAST_REQUIRED_HEADERS="$REQUIRED_HEADERS"
-
- # remove the file that stores all required headers line by line
- rm -f tmp_header_file
-
- for HEADER in $REQUIRED_HEADERS; do
- # if tmp_header_file does not exist, create it
- if test "$(ls tmp_header_file 2>/dev/null)" = ""; then
- touch tmp_header_file
- fi
-
- # append header name to tmp_header_file
- echo "$HEADER" >> tmp_header_file
-
- # obtain the base name and the directory name of the header that is considered
- HEADER_BASE_NAME=$(basename $HEADER)
- HEADER_DIR_NAME=$(dirname $HEADER)
-
- # create a list of all files that are included from the header
- sed -i 's/#include/#include /' $HEADER
- INCLUDE_LIST=$(tr -s " " < $HEADER | tr -d '><"' | awk '/#include/{print $2}')
- sed -i 's/#include /#include/' $HEADER
-
- # look at each of the included files
- for INCLUDED_HEADER in $INCLUDE_LIST; do
- # don't include config.h
- if test "$INCLUDED_HEADER" = "config.h"; then
- continue
- fi
-
- INCLUDED_BASE_NAME=$(basename $INCLUDED_HEADER)
- INCLUDED_DIR_NAME=$(dirname $INCLUDED_HEADER)
-
- # if the header file exists, add it
- if test "$(ls $INCLUDED_HEADER 2>/dev/null)" = "$INCLUDED_HEADER"; then
- echo "$INCLUDED_HEADER" >> tmp_header_file
- continue
- fi
-
- # try the header preceded by its path
- INCLUDED_HEADER_WITH_PATH="${HEADER_DIR_NAME}/${INCLUDED_HEADER}"
-
- # if the header file actually exists, add it
- if test "$(ls $INCLUDED_HEADER_WITH_PATH 2>/dev/null)" = "$INCLUDED_HEADER_WITH_PATH"; then
- # remove "../" from the path
- cd $(dirname $INCLUDED_HEADER_WITH_PATH)
- HEADER_FULL_PATH=$(pwd)
- HEADER_RELATIVE_PATH=${HEADER_FULL_PATH#$MODULE_FULL_PATH}
- HEADER_RELATIVE_PATH=$(echo $HEADER_RELATIVE_PATH | sed 's/^.//')
- INCLUDED_HEADER_WITH_PATH="${HEADER_RELATIVE_PATH}/${INCLUDED_BASE_NAME}"
- cd $MODULE_FULL_PATH
- echo "$INCLUDED_HEADER_WITH_PATH" >> tmp_header_file
- fi
- done
- done
-
- # sort the required headers, eliminate copies
- REQUIRED_HEADERS=$(sort -u tmp_header_file)
-
-done
-
-# remove the file that stores all required headers line by line
-rm -f tmp_header_file
-
-# provide some output and copy everything to the new module
-echo ""
-echo -n "Number of required headers: "
-echo "$REQUIRED_HEADERS" | wc -w
-for HEADER in $REQUIRED_HEADERS; do
- echo $HEADER
- rsync -R $HEADER ../$MODULE_NAME
-done
-for DIR_PATH in $ALL_DIRECTORIES; do
- rsync -rR $DIR_PATH ../$MODULE_NAME
-done
-
-# delete all architecture-dependent files
-cd ../$MODULE_NAME
-find . -name Makefile.in -delete
-find . -name Makefile -delete
-find . -name '*.o' -delete
-find . -wholename '*.deps/*' -delete
-find . -path '*.deps' -delete
-echo "Removed architecture-dependent files."
-
-# remove directories that are not required
-sed -i '/(dune)/d' CMakeLists.txt
-sed -i '/(src)/d' CMakeLists.txt
-rm -rf dune/
-rm -rf src/
-
-# create a list of the subfolders
-EMPTY_DIR_NAME="."
-rm -f tmp_subdir_file
-for HEADER in $REQUIRED_HEADERS; do
-
- # move through every header, cutting off the last folder
- while test "$HEADER" != $EMPTY_DIR_NAME; do
- if test "$(dirname $HEADER)" != $EMPTY_DIR_NAME; then
- dirname $HEADER >> tmp_subdir_file
- fi
-
- HEADER=$(dirname $HEADER)
- done
-
- # finally add the topmost folder
- if test "$(basename $HEADER)" != $EMPTY_DIR_NAME; then
- basename $HEADER >> tmp_subdir_file
- fi
-done
-SUBDIR_LIST=$(sort -u tmp_subdir_file)
-rm -f tmp_subdir_file
-
-# treat every subfolder
-START_DIR=$PWD
-INITIAL_SUBDIR_LIST=""
-for SUBDIR in $SUBDIR_LIST; do
- if test "$SUBDIR" = "$(basename $SUBDIR)"; then
- INITIAL_SUBDIR_LIST="$INITIAL_SUBDIR_LIST $SUBDIR"
- fi
-
- cd $SUBDIR
-
- # create lists of files and folders
- FILE_LIST=""
- DIR_LIST=""
- for FILE_OR_FOLDER in $( ls ); do
- if ([ -f $FILE_OR_FOLDER ]); then
- FILE_LIST="$FILE_LIST $FILE_OR_FOLDER"
- fi
-
- if ([ -d $FILE_OR_FOLDER ]); then
- DIR_LIST="$DIR_LIST $FILE_OR_FOLDER"
- fi
- done
-
- # create CMakeLists.txt if not present
- if ([ ! -f CMakeLists.txt ]); then
- # add subfolder information
- for DIR in $DIR_LIST; do
- echo "add_subdirectory($DIR)" >> CMakeLists.txt
- done
-
- # determine whether to add file information
- ADD_FILE_INFO="0"
- for FILE in $FILE_LIST; do
- if ([ -f $FILE ]); then
- ADD_FILE_INFO="1"
- fi
- done
-
- # add file information
- if test "$ADD_FILE_INFO" = "1"; then
- echo "" >> CMakeLists.txt
- echo "install(FILES" >> CMakeLists.txt
-
- for FILE in $FILE_LIST; do
- echo " $FILE" >> CMakeLists.txt
- done
-
- echo " DESTINATION \${CMAKE_INSTALL_INCLUDEDIR}/$SUBDIR)" >> CMakeLists.txt
- fi
- fi
-
- cd $START_DIR
-
-done
-
-# update top CMakeLists.txt
-for INITIAL_SUBDIR in $INITIAL_SUBDIR_LIST; do
- sed -i '/add_subdirectory(doc)/a add_subdirectory('$INITIAL_SUBDIR')' CMakeLists.txt
-done
-echo "Updated and created CMakeLists.txt's."
-cd ..
-
-# add includes to the automatically generated file *Macros.cmake
-MACROS_FILE="$(find $MODULE_NAME -name '*Macros.cmake')"
-if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_csv_file_links {} \; 2>/dev/null)" != ""; then
- cp dumux-devel/cmake/modules/AddCSVFileLinks.cmake $MODULE_NAME/cmake/modules/.
- echo "include(AddCSVFileLinks)" >>$MACROS_FILE
-fi
-if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_executable_all {} \; 2>/dev/null)" != ""; then
- cp dumux-devel/cmake/modules/AddExecutableAll.cmake $MODULE_NAME/cmake/modules/.
- echo "include(AddExecutableAll)" >>$MACROS_FILE
-fi
-if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_file_link {} \; 2>/dev/null)" != ""; then
- cp dumux-devel/cmake/modules/AddFileLink.cmake $MODULE_NAME/cmake/modules/.
- echo "include(AddFileLink)" >>$MACROS_FILE
-fi
-if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_folder_link {} \; 2>/dev/null)" != ""; then
- cp dumux-devel/cmake/modules/AddFolderLink.cmake $MODULE_NAME/cmake/modules/.
- echo "include(AddFolderLink)" >>$MACROS_FILE
-fi
-if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_gnuplot_file_links {} \; 2>/dev/null)" != ""; then
- cp dumux-devel/cmake/modules/AddGnuplotFileLinks.cmake $MODULE_NAME/cmake/modules/.
- echo "include(AddGnuplotFileLinks)" >>$MACROS_FILE
-fi
-
-# create $README_FILE
-README_FILE="README.md"
-mv $MODULE_NAME/README $MODULE_NAME/$README_FILE
-echo "This file has been created automatically. Please adapt it to your needs." >$MODULE_NAME/$README_FILE
-echo "" >>$MODULE_NAME/$README_FILE
-echo "===============================" >>$MODULE_NAME/$README_FILE
-echo "" >>$MODULE_NAME/$README_FILE
-echo "## Content" >>$MODULE_NAME/$README_FILE
-echo "" >>$MODULE_NAME/$README_FILE
-echo "The content of this DUNE module was extracted from the module \`$MODULE_DIR\`." >>$MODULE_NAME/$README_FILE
-echo "In particular, the following subfolders of \`$MODULE_DIR\` have been extracted:" >>$MODULE_NAME/$README_FILE
-echo "" >>$MODULE_NAME/$README_FILE
-for DIR_PATH in $ALL_DIRECTORIES; do
- echo "* \`$DIR_PATH\`," >>$MODULE_NAME/$README_FILE
-done
-echo "" >>$MODULE_NAME/$README_FILE
-echo "Additionally, all headers in \`$MODULE_DIR\` that are required to build the" >>$MODULE_NAME/$README_FILE
-echo "executables from the sources" >>$MODULE_NAME/$README_FILE
-echo "" >>$MODULE_NAME/$README_FILE
-for SOURCE in $ALL_SOURCES; do
- echo "* \`$SOURCE\`," >>$MODULE_NAME/$README_FILE
-done
-echo "" >>$MODULE_NAME/$README_FILE
-echo "have been extracted. You can configure the module just like any other DUNE" >>$MODULE_NAME/$README_FILE
-echo "module by using \`dunecontrol\`. For building and running the executables," >>$MODULE_NAME/$README_FILE
-echo "please go to the build folders corresponding to the sources listed above." >>$MODULE_NAME/$README_FILE
-echo "" >>$MODULE_NAME/$README_FILE
-
-# get versions from modules we are depeding on
-# create patches for un-pushed commits and uncomitted changes
-
-# try to find the dunecontrol script
-if hash dunecontrol 2>/dev/null; then
- DUNE_CONTROL="dunecontrol"
-else
- DUNE_CONTROL=$(find . -type f -executable -name 'dunecontrol')
-fi
-if test "$DUNE_CONTROL" = ""; then
- echo "ERROR: Could not find dunecontrol."
- echo "Be sure to either have dunecontrol in your search path"
- echo "or to run this script from a directory that contains dunecontrol."
- exit
-fi
-# get names of all modules we are depeding on
-echo "Extracting dependencies to other Dune modules"
-DEPENDING_MODULE_NAMES=$($DUNE_CONTROL --module=$MODULE_NAME 2>/dev/null | head -n 1)
-DEPENDING_MODULE_NAMES=$(sed 's/--- going to build //' <<< "$DEPENDING_MODULE_NAMES")
-DEPENDING_MODULE_NAMES=$(sed 's/ ---//' <<< "$DEPENDING_MODULE_NAMES")
-# load script
-CALL_FROM_EXTERNAL_SCRIPT="yes"
-SCRIPT_DIR="${BASH_SOURCE%/*}"
-if [[ ! -d "$SCRIPT_DIR" ]]; then
- SCRIPT_DIR="$PWD";
-fi
-
-# output guidence for users
-echo ""
-echo "*************************************************************************"
-echo "The extracted module is contained in the subfolder \"$MODULE_NAME\"."
-echo "You can build it using \"dunecontrol ... --only=$MODULE_NAME all\"."
-echo "*************************************************************************"
-echo "BEFORE building, you can add the module to dumux-pub by something like:"
-echo "(Rename module name if it does not match the AuthorLastNameYearx scheme"
-echo "and commit it to the Git repository dumux-pub)"
-echo "git clone https://git.iws.uni-stuttgart.de/dumux-pub/AuthorLastNameYearx.git"
-echo "sed -i '/Module:/c\Module: AuthorLastNameYearx' $MODULE_NAME/dune.module"
-echo "mv $MODULE_NAME/* dumux-pub/AuthorLastNameYearx/."
-echo "cd AuthorLastNameYearx"
-echo "git commit -a"
-echo "git push"
-
-exit 0
diff --git a/bin/util/getusedversions.py b/bin/util/getusedversions.py
deleted file mode 100755
index 18b407051a3695fbf3cd86b9059f1ce2dcdb5e1e..0000000000000000000000000000000000000000
--- a/bin/util/getusedversions.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import argparse
-from common import runCommand
-from common import callFromPath
-
-# print warning message for scanned folders that are not git repositories
-def printNoGitRepoWarning(folderPath):
- print("Folder " + folderPath + " does not seem to be the top level" \
- "of a git repository and will be skipped. Make sure not to call " \
- "this script from a sub-directory of a git repository.")
-
-# raise error due to untracked files present in the given module folder
-def raiseUntrackedFilesError(folderPath):
- raise RuntimeError('Found untracked files in module folder: "' + folderPath + '". ' \
- 'Please commit, stash, or remove them.')
-
-# returns true if the given folder is a git repository
-def isGitRepository(modFolderPath):
- return os.path.exists(os.path.join(modFolderPath, '.git'))
-
-# returns true if a module contains untracked files
-def hasUntrackedFiles(modFolderPath):
- run = callFromPath(modFolderPath)(runCommand)
- return run('git ls-files --others --exclude-standard') != ''
-
-# function to extract git version information for modules
-# returns a dictionary containing module information for each given module folder
-def getUsedVersions(modFolderPaths, ignoreUntracked=False):
-
- result = {}
- for modFolderPath in modFolderPaths:
-
- # make sure this is the top level of a git repository
- if not isGitRepository(modFolderPath):
- printNoGitRepoWarning(modFolderPath)
- else:
- if not ignoreUntracked and hasUntrackedFiles(modFolderPath):
- raiseUntrackedFilesError(modFolderPath)
-
- run = callFromPath(modFolderPath)(runCommand)
- result[modFolderPath] = {}
- result[modFolderPath]['remote'] = run('git ls-remote --get-url').strip('\n')
- result[modFolderPath]['revision'] = run('git log -n 1 --format=%H @{upstream}').strip('\n')
- result[modFolderPath]['date'] = run('git log -n 1 --format=%ai @{upstream}').strip('\n')
- result[modFolderPath]['author'] = run('git log -n 1 --format=%an @{upstream}').strip('\n')
- result[modFolderPath]['branch'] = run('git rev-parse --abbrev-ref HEAD').strip('\n')
-
- return result
-
-# create patches for unpublished commits and uncommitted changes in modules
-def getPatches(modFolderPaths, ignoreUntracked=False):
-
- result = {}
- for modFolderPath in modFolderPaths:
-
- # make sure this is the top level of a git repository
- if not isGitRepository(modFolderPath):
- printNoGitRepoWarning(modFolderPath)
- else:
- if not ignoreUntracked and hasUntrackedFiles(modFolderPath):
- raiseUntrackedFilesError(modFolderPath)
-
- run = callFromPath(modFolderPath)(runCommand)
- unpubPatch = run('git format-patch --stdout @{upstream}')
- unCommPatch = run('git diff')
- if unpubPatch != '' or unCommPatch != '': result[modFolderPath] = {}
- if unpubPatch != '': result[modFolderPath]['unpublished'] = unpubPatch
- if unCommPatch != '': result[modFolderPath]['uncommitted'] = unCommPatch
-
- return result
-
-# prints the detected versions as table
-def printVersionTable(versions):
- print("\t| {:^50} | {:^50} | {:^50} | {:^30} |".format('module folder', 'branch', 'commit hash', 'commit date'))
- print("\t" + 193*'-')
- for folder, versionInfo in versions.items():
- print("\t| {:^50} | {:^50} | {:^50} | {:^30} |".format(folder, versionInfo['branch'], versionInfo['revision'], versionInfo['date']))
-
-
-# For standalone execution
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description='This script extracts the used dune/dumux versions.')
- parser.add_argument('-p', '--path', required=False, help='the path to the top folder containing your dune/dumux modules')
- parser.add_argument('-i', '--ignoreuntracked', required=False, action='store_true', help='use this flag to ignore untracked files present in the modules')
- parser.add_argument('-s', '--skipfolders', required=False, nargs='*', help='a list of module folders to be skipped')
- cmdArgs = vars(parser.parse_args())
-
- modulesPath = os.getcwd() if not cmdArgs['path'] else os.path.join(os.getcwd(), cmdArgs['path'])
- print('\nDetermining the versions of all dune modules in the folder: ' + modulesPath)
-
- def getPath(modFolder):
- return os.path.join(modulesPath, modFolder)
-
- modFolderPaths = [getPath(dir) for dir in os.listdir(modulesPath) if os.path.isdir(getPath(dir))]
- if cmdArgs['skipfolders']:
- cmdArgs['skipfolders'] = [f.strip('/') for f in cmdArgs['skipfolders']]
- modFolderPaths = [d for d in modFolderPaths if os.path.basename(d.strip('/')) not in cmdArgs['skipfolders']]
- versions = getUsedVersions(modFolderPaths, True)
-
- print("\nDetected the following versions:")
- printVersionTable(versions)
-
- # maybe check untracked files
- if not cmdArgs['ignoreuntracked']:
- modsWithUntracked = [f for f in versions if hasUntrackedFiles(f)]
- if modsWithUntracked:
- print('\n')
- print('#'*56)
- print('WARNING: Found untracked files in the following modules:\n\n')
- print('\n'.join(modsWithUntracked))
- print('\nPlease make sure that these are not required for your purposes.')
- print('If not, you can run this script with the option -i/--ignoreuntracked to suppress this warning.')
- print('#'*56)
diff --git a/bin/util/installscript.py b/bin/util/installscript.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d8669d75b0c71c9d77e6735db1865532e226e58
--- /dev/null
+++ b/bin/util/installscript.py
@@ -0,0 +1,203 @@
+""""
+Helper functions to generate an install script for a dune-module,
+accounting for non-published commits and local changes
+"""
+
+import os
+import sys
+import textwrap
+
+from util.common import getPersistentVersions, versionTable, getPatches
+from util.moduleinfo import getModuleInfo
+from util.installscript_writer import InstallScriptWriterBash
+from util.installscript_writer import InstallScriptWriterPython
+
+if sys.version_info[0] < 3:
+ sys.exit("\nError': Python3 required")
+
+
+def supportedLanguages():
+ return ['python', 'bash']
+
+
+def getScriptExtension(language):
+ assert language in supportedLanguages()
+ ext = {
+ 'python': '.py',
+ 'bash': '.sh'
+ }
+ return ext[language]
+
+
+def makeScriptWriter(language):
+ if language == 'bash':
+ return InstallScriptWriterBash()
+ elif language == 'python':
+ return InstallScriptWriterPython()
+ raise ValueError(f'Could not create writer for language {language}')
+
+
+def getDefaultScriptName(modName, language):
+ return 'install_{}{}'.format(
+ modName,
+ getScriptExtension(language)
+ )
+
+
+def printProgressInfo(infoLines, indLevel=0):
+ firstPrefix = '\n' + '--'*(indLevel+1)
+ emptyPrefix = firstPrefix.replace('-', ' ').strip('\n')
+ print(f"{firstPrefix} {infoLines[0]}")
+ for line in infoLines[1:]:
+ print(f"{emptyPrefix} {line}")
+
+
+def filterDependencies(dependencies, skipFolders=[]):
+ if not skipFolders:
+ return dependencies
+ else:
+ def skipFolder(folderName):
+ return any(folderName == os.path.basename(path) for path in skipFolders)
+ return [
+ dep for dep in dependencies if not skipFolder(dep['folder'])
+ ]
+
+
+def addDependencyVersions(dependencies, ignoreUntracked=False):
+ def getKey(dependency):
+ return dependency['path']
+
+ versions = getPersistentVersions(
+ [getKey(d) for d in dependencies], ignoreUntracked
+ )
+ if len(versions) != len(dependencies):
+ raise Exception("Not all versions of all modules could be found.")
+
+ mergedResult = []
+ for depInfo in dependencies:
+ versionInfo = versions[getKey(depInfo)]
+ mergedResult.append({**depInfo, **versionInfo})
+ return mergedResult
+
+
+def addDependencyPatches(dependenciesWithVersions):
+ def getKey(dependency):
+ return dependency['path']
+
+ patches = getPatches({
+ getKey(d): d for d in dependenciesWithVersions
+ })
+
+ mergedResult = []
+ for depInfo in dependenciesWithVersions:
+ patch = patches[getKey(depInfo)]
+ mergedResult.append({**depInfo, **patch})
+ return mergedResult
+
+
+def makeInstallScript(modPath,
+ dependencies,
+ scriptName,
+ writer,
+ topFolderName='DUMUX',
+ optsFile=None):
+
+ modPath = os.path.abspath(modPath)
+ modName = getModuleInfo(modPath, 'Module')
+
+ modOptsFile = '{}/cmake.opts'.format(modPath)
+ if not optsFile:
+ if os.path.isfile(modOptsFile):
+ optsFile = '{}/cmake.opts'.format(os.path.relpath(modPath))
+ else:
+ optsFile = 'dumux/cmake.opts'
+ if os.path.isabs(optsFile):
+ raise ValueError("Opts file must be given as relative path")
+ if not any(optsFile.startswith(d['folder']) for d in dependencies):
+ print("Warning: opts file is not contained in any of the dependencies")
+
+ with open(scriptName, 'w') as script:
+
+ writer.setOutputStream(script)
+ writer.writeSheBang()
+
+ script.write('\n')
+ writer.writeComment(textwrap.dedent(f"""\
+
+ This installs the module {modName} and its dependencies.
+ The exact revisions used are listed in the table below.
+ However, note that this script may also apply further patches.
+ If so, all patches are required to be the current folder, or,
+ in the one that you specified as argument to this script.
+
+ """))
+
+ script.write('\n')
+ writer.writeComment(versionTable(dependencies))
+
+ script.write('\n')
+ writer.writePreamble(topFolderName)
+
+ for dep in dependencies:
+ script.write('\n')
+ writer.writeMessageOutput('Installing {}'.format(dep['name']))
+ writer.writeInstallation(dep)
+
+ for dep in dependencies:
+ def writePatch(patch, moduleName, description):
+ script.write('\n')
+ writer.writeMessageOutput(
+ f'Applying patch for {description} in {moduleName}'
+ )
+ writer.writePatchApplication(dep['folder'], patch)
+
+ if dep['untracked'] is not None:
+ description = 'untracked files'
+ writePatch(dep['untracked'], description, dep['name'])
+ if dep['unpublished'] is not None:
+ description = 'unpublished commits'
+ writePatch(dep['unpublished'], description, dep['name'])
+ if dep['uncommitted'] is not None:
+ description = 'uncommitted changes'
+ writePatch(dep['uncommitted'], description, dep['name'])
+
+ script.write('\n')
+ writer.writeMessageOutput('Configuring project')
+ writer.writeConfiguration(optsFile)
+
+
+def printFoundDependencies(deps):
+ if len(deps) > 0:
+ infoText = ["Found the following dependencies"]
+ infoText.extend(
+ versionTable(
+ deps, {'name': 'module name', 'path': 'folder'}
+ ).split('\n')
+ )
+ printProgressInfo(infoText)
+
+
+def printFoundVersionInfo(dependenciesWithVersions):
+ table = versionTable(dependenciesWithVersions)
+ printProgressInfo(
+ ["The following (remotely available) versions are used as a basis",
+ "on top of which the required patches will be automatically created:",
+ "\n{}".format(table)]
+ )
+
+
+def printFinalMessage(scriptName,
+ topFolderName=None):
+
+ if topFolderName:
+ description = textwrap.dedent(f"""\
+ Running this script will create a folder `{topFolderName}`, clone all modules
+ into it, configure the entire project and build the contained applications
+ """)
+ else:
+ description = textwrap.dedent(f"""\
+ Running this script will clone all modules into the folder from which it is
+ called, configure the entire project and build the contained applications
+ """)
+
+ printProgressInfo(['Info:', description])
diff --git a/bin/util/installscript_writer.py b/bin/util/installscript_writer.py
new file mode 100644
index 0000000000000000000000000000000000000000..0eb133b344eda1eab734e393a49bfbad913b5271
--- /dev/null
+++ b/bin/util/installscript_writer.py
@@ -0,0 +1,222 @@
+"""Language-specific backends for install script generation"""
+
+import os
+import sys
+import textwrap
+from abc import ABC, abstractmethod
+from util.common import addPrefixToLines, escapeCharacters
+
+
+def getRawString(text):
+ def makeRaw(text): return repr(text)
+ def removeEnclosingQuotes(text): return text[1:-1]
+ return removeEnclosingQuotes(makeRaw(text))
+
+
+class InstallScriptWriterInterface(ABC):
+ def __init__(self):
+ self.ostream = None
+
+ def setOutputStream(self, stream):
+ self.ostream = stream
+
+ @abstractmethod
+ def writeSheBang(self):
+ pass
+
+ @abstractmethod
+ def writeComment(self, comment):
+ pass
+
+ @abstractmethod
+ def writeMessageOutput(self, message):
+ pass
+
+ @abstractmethod
+ def writePreamble(self, topFolderName=None):
+ pass
+
+ @abstractmethod
+ def writeInstallation(self, dependency):
+ pass
+
+ @abstractmethod
+ def writePatchApplication(self, folder, patchName):
+ pass
+
+ @abstractmethod
+ def writeConfiguration(self, optsFile):
+ pass
+
+
+class InstallScriptWriterBash(InstallScriptWriterInterface):
+ def __init__(self):
+ super().__init__()
+
+ def writeSheBang(self):
+ self.ostream.write('#!/bin/bash\n')
+
+ def writeComment(self, comment):
+ comment = addPrefixToLines('#', comment)
+ self.ostream.write(comment)
+
+ def writeMessageOutput(self, message):
+ self.ostream.write(f'echo "{message}"\n')
+
+ def writePreamble(self, topFolderName=None):
+ self.ostream.write(textwrap.dedent("""\
+
+ exitWithError()
+ {
+ MSG=$1
+ echo "$MSG"
+ exit 1
+ }
+
+ installModule()
+ {
+ FOLDER=$1
+ URL=$2
+ BRANCH=$3
+ REVISION=$4
+
+ if [ ! -d "$FOLDER" ]; then
+ if ! git clone $URL; then exitWithError "clone failed"; fi
+ pushd $FOLDER
+ if ! git checkout $BRANCH; then exitWithError "checkout failed"; fi
+ if ! git reset --hard $REVISION; then exitWithError "reset failed"; fi
+ popd
+ else
+ echo "Skip cloning $URL since target folder "$FOLDER" already exists."
+ fi
+ }
+
+ applyPatch()
+ {
+ FOLDER=$1
+ PATCH=$2
+
+ pushd $FOLDER
+ echo "$PATCH" > tmp.patch
+ if ! git apply tmp.patch; then exitWithError "patch failed"; fi
+ rm tmp.patch
+ popd
+ }
+
+ """))
+ top = topFolderName if topFolderName else "."
+ self.ostream.write('TOP="{}"\n'.format(top))
+ self.ostream.write('mkdir -p $TOP\n')
+ self.ostream.write('cd $TOP\n')
+
+ def writeInstallation(self, dependency):
+ self.ostream.write('installModule {} {} {} {}'
+ .format(dependency['folder'],
+ dependency['remote'],
+ dependency['branch'],
+ dependency['revision']))
+
+ def writePatchApplication(self, folder, patchContent):
+ def removeEscapedSingleQuotes(line):
+ return line.replace(r"\'", "'")
+
+ self.ostream.write('PATCH="\n')
+ for line in patchContent.rstrip('\n').split('\n'):
+ line = getRawString(line)
+ line = removeEscapedSingleQuotes(line)
+ line = escapeCharacters(line, ['"', "$", "`"])
+ self.ostream.write(line)
+ self.ostream.write('\n')
+ self.ostream.write('"\n')
+ self.ostream.write(f'applyPatch {folder} "$PATCH"')
+
+ def writeConfiguration(self, opts):
+ self.ostream.write(
+ f'if ! ./dune-common/bin/dunecontrol --opts={opts} all; then\n'
+ ' echo "Configuration of the project failed"\n'
+ ' exit 1\n'
+ 'fi\n'
+ )
+
+
+class InstallScriptWriterPython(InstallScriptWriterInterface):
+ def __init__(self):
+ super().__init__()
+
+ def writeSheBang(self):
+ self.ostream.write('#!/usr/bin/env python3\n')
+
+ def writeComment(self, comment):
+ comment = addPrefixToLines('#', comment)
+ self.ostream.write(comment)
+
+ def writeMessageOutput(self, message):
+ self.ostream.write(f'print("{message}")\n')
+
+ def writePreamble(self, topFolderName=None):
+ top = topFolderName if topFolderName else "."
+ self.ostream.write(textwrap.dedent(f"""\
+
+ import os
+ import sys
+ import subprocess
+
+ top = "{top}"
+ os.makedirs(top, exist_ok=True)
+
+
+ def runFromSubFolder(cmd, subFolder):
+ folder = os.path.join(top, subFolder)
+ try:
+ subprocess.run(cmd, cwd=folder, check=True)
+ except Exception as e:
+ cmdString = ' '.join(cmd)
+ sys.exit(
+ "Error when calling:\\n{{}}\\n-> folder: {{}}\\n-> error: {{}}"
+ .format(cmdString, folder, str(e))
+ )
+
+
+ def installModule(subFolder, url, branch, revision):
+ targetFolder = url.rstrip(".git").split("/")[-1]
+ if not os.path.exists(targetFolder):
+ runFromSubFolder(['git', 'clone', url, targetFolder], '.')
+ runFromSubFolder(['git', 'checkout', branch], subFolder)
+ runFromSubFolder(['git', 'reset', '--hard', revision], subFolder)
+ else:
+ print(f'Skip cloning {{url}} since target folder "{{targetFolder}}" already exists.')
+
+
+ def applyPatch(subFolder, patch):
+ sfPath = os.path.join(top, subFolder)
+ patchPath = os.path.join(sfPath, 'tmp.patch')
+ with open(patchPath, 'w') as patchFile:
+ patchFile.write(patch)
+ runFromSubFolder(['git', 'apply', 'tmp.patch'], subFolder)
+ os.remove(patchPath)
+ """))
+
+ def writeInstallation(self, dependency):
+ self.ostream.write('installModule("{}", "{}", "{}", "{}")\n'
+ .format(dependency['folder'],
+ dependency['remote'],
+ dependency['branch'],
+ dependency['revision']))
+
+ def writePatchApplication(self, folder, patchContent):
+ self.ostream.write('patch = """\n')
+ for line in patchContent.rstrip('\n').split('\n'):
+ line = getRawString(line)
+ self.ostream.write(escapeCharacters(line, ['"']))
+ self.ostream.write('\n')
+ self.ostream.write('"""\n')
+
+ self.ostream.write(f'applyPatch("{folder}", patch)\n')
+
+ def writeConfiguration(self, opts):
+ self.ostream.write(
+ "runFromSubFolder(\n"
+ f" ['./dune-common/bin/dunecontrol', '--opts={opts}', 'all'],\n"
+ " '.'\n"
+ ")\n"
+ )
diff --git a/bin/util/makeinstallscript.py b/bin/util/makeinstallscript.py
deleted file mode 100755
index 83248590051d82e68b4c45fc7270fec341ea9f13..0000000000000000000000000000000000000000
--- a/bin/util/makeinstallscript.py
+++ /dev/null
@@ -1,250 +0,0 @@
-#!/usr/bin/env python3
-
-# script to generate an install script for a dune-module,
-# accounting for non-published commits and local changes
-
-import os
-import sys
-import argparse
-import subprocess
-from getmoduleinfo import *
-from getusedversions import *
-
-if sys.version_info[0] < 3:
- sys.exit("\nError': Python3 required")
-
-
-###################
-# parse arguments
-parser = argparse.ArgumentParser(
- description='This script generates an install script for your dune module,' \
- 'taking into account non-published commits and local changes.\n' \
- 'This expects that your module is a git repository and that a ' \
- 'remote origin exists and has been set already.'
-)
-parser.add_argument('-p', '--path', required=True, help='The path to your dune module')
-parser.add_argument('-f', '--filename', required=False, help='Name of the file in which to write the install script')
-parser.add_argument('-i', '--ignoreuntracked', required=False, action='store_true', help='Use this flag to ignore untracked files present in the modules')
-parser.add_argument('-t', '--topfoldername', required=False, default='DUMUX',
- help='Name of the folder that the install script creates upon execution to install the modules in. '\
- 'If you pass an empty string, no folder will be created and installation happens in place.')
-parser.add_argument('-o', '--optsfile', required=False,
- help='Provide custom opts file to be used for the call to dunecontrol. '\
- 'Note that this file is required to be contained and committed within the module or its dependencies.')
-parser.add_argument('-s', '--skipfolders', required=False, nargs='*', help='a list of module folders to be skipped')
-cmdArgs = vars(parser.parse_args())
-
-
-####################
-# Welcome message
-modPath = os.path.abspath( os.path.join(os.getcwd(), cmdArgs['path']) )
-modParentPath = os.path.abspath( os.path.join(modPath, '../') )
-modFolder = os.path.relpath(modPath, modParentPath)
-
-try: modName = getModuleInfo(modPath, 'Module')
-except:
- sys.exit("\Error: Could not determine module name. Make sure the path to\n" \
- " your module is correct and that a dune.module file is present.")
-
-instFileName = 'install_' + modName + '.sh' if not cmdArgs['filename'] else cmdArgs['filename']
-print("\n-- Creating install script '{}' for the module '{}' in the folder '{}'".format(instFileName, modName, modPath))
-
-
-###################
-# get dependencies
-print("\n-- Determining the dependencies")
-deps = getDependencies(modPath)
-if len(deps) > 0:
- print("-> Found the following dependencies")
- print("\t| {:^50} | {:^50} |".format('module name', 'module folder'))
- print("\t" + 107*'-')
- for dep in deps: print('\t| {:^50} | {:^50} |'.format(dep['name'], dep['folder']))
-else:
- sys.exit("Error: Could not determine dependencies. At least the module itself should appear.")
-
-depNames = [dep['name'] for dep in deps]
-depFolders = [dep['folder'] for dep in deps]
-depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders]
-if cmdArgs['skipfolders']:
- cmdArgs['skipfolders'] = [f.strip('/') for f in cmdArgs['skipfolders']]
- depFolders = [d for d in depFolders if d not in cmdArgs['skipfolders']]
- depNames = [getModuleInfo(d, 'Module') for d in depFolders]
- depFolderPaths = [d for d in depFolderPaths if os.path.basename(d.strip('/')) not in cmdArgs['skipfolders']]
-
-#################################
-# determine specific commits of all modules
-print("\n-- Determining the module versions")
-try: versions = getUsedVersions(depFolderPaths, cmdArgs['ignoreuntracked'])
-except Exception as err:
- print('\nCaught exception: ' + str(err))
- if 'untracked' in str(err):
- print('If you are certain that the untracked files are not needed for the ' \
- 'installation of the modules, run this script with the -i flag.')
- sys.exit(1)
-
-if len(versions) != len(depNames):
- sys.exit("Error': Could not determine versions of all modules")
-
-print("-> Detected the following versions")
-printVersionTable(versions)
-
-#################################
-# create patches if necessary
-print("\n-- Creating patches for unpublished commits and uncommitted changes")
-patches = getPatches(depFolderPaths, cmdArgs['ignoreuntracked'])
-
-# create patch files
-if len(patches) > 0:
- patchesPath = os.path.join(modPath, 'patches')
- if not os.path.exists(patchesPath):
- os.mkdir(patchesPath)
- print("-> Created a folder 'patches' in your module. You should commit this to your\n"\
- " repository in order for the installation script to work on other machines.")
- else:
- print("-> Adding patches to the 'patches' folder in your module. Make sure to commit\n"\
- " the newly added patches in order for the install script to work on other machines.")
-
- # get a non-used patch file name (makes sure not to overwrite anything)
- def getPatchFileName(depModPath, targetName):
- i = 1
- fileName = os.path.join(patchesPath, targetName + '.patch')
- while os.path.exists(fileName):
- fileName = os.path.join(patchesPath, targetName + '_' + str(i) + '.patch')
- i += 1
-
- return fileName, os.path.relpath(fileName, depModPath)
-
- # function to write a new patch file (returns the path to the new file)
- def writeDepModPatchFile(depModPath, depModName, type):
- patchPath, patchRelPath = getPatchFileName(depModPath, depModName + '_' + type)
- patches[depModPath][type + '_relpath'] = patchRelPath
- open(patchPath, 'w').write(patches[depModPath][type])
- return patchPath
-
- print("-> Created patch files:")
- for depModPath in patches.keys():
- depModName = getModuleInfo(depModPath, "Module")
- if 'unpublished' in patches[depModPath]:
- print(' '*3 + writeDepModPatchFile(depModPath, depModName, 'unpublished'))
- if 'uncommitted' in patches[depModPath]:
- print(' '*3 + writeDepModPatchFile(depModPath, depModName, 'uncommitted'))
-
-else:
- print("-> No Patches required")
-
-
-##################################
-# write installation shell script
-
-# in the install script, we work with relative paths from the parent to module folder
-def getModRelPath(path): return os.path.relpath(path, modParentPath)
-versions = {getModRelPath(modPath): val for modPath, val in versions.items()}
-patches = {getModRelPath(modPath): val for modPath, val in patches.items()}
-
-topFolderName = cmdArgs['topfoldername']
-optsRelPath = 'dumux/cmake.opts'
-if cmdArgs['optsfile']:
- optsPath = os.path.abspath( os.path.join(os.getcwd(), cmdArgs['optsfile']) )
- optsRelPath = os.path.relpath(optsPath, modParentPath)
-
-# TODO: add support for different install script languages (e.g. Python).
-with open(instFileName, 'w') as installFile:
- installFile.write('#!/bin/bash\n\n')
- installFile.write('#'*80 + '\n')
- installFile.write('# This script installs the module "' + modName + '" together with all dependencies.\n')
- installFile.write('\n')
-
- exitFunc = 'exitWith'
- installFile.write('# defines a function to exit with error message\n')
- installFile.write(exitFunc + ' ()\n')
- installFile.write('{\n')
- installFile.write(' echo ' + r'"\n$1"' +'\n')
- installFile.write(' exit 1\n')
- installFile.write('}\n\n')
-
- # function to write a command with error check into the script file
- def writeCommand(command, errorMessage, indentationLevel = 0):
- installFile.write(' '*indentationLevel + 'if ! ' + command + ';')
- installFile.write(' then ' + exitFunc + ' "' + errorMessage + '"; fi\n')
-
- # write section on creating the top folder
- if topFolderName:
- installFile.write('# Everything will be installed into a newly created sub-folder named "' + topFolderName + '".\n')
- installFile.write('echo "Creating the folder ' + topFolderName + ' to install the modules in"\n')
- writeCommand('mkdir -p ' + topFolderName , '--Error: could not create top folder ' + topFolderName)
- writeCommand('cd ' + topFolderName, '--Error: could not enter top folder ' + topFolderName)
- installFile.write('\n')
- else:
- installFile.write('# Everything will be installed inside the folder from which the script is executed.\n\n')
-
- # function to write installation procedure for a module
- def writeCloneModule(depModName, depModFolder):
- installFile.write('# ' + depModName + '\n')
- installFile.write('# ' + versions[depModFolder]['branch'] + ' # '
- + versions[depModFolder]['revision'] + ' # '
- + versions[depModFolder]['date'] + ' # '
- + versions[depModFolder]['author'] + '\n')
-
- writeCommand('git clone ' + versions[depModFolder]['remote'],
- '-- Error: failed to clone ' + depModName + '.')
- writeCommand('cd ' + depModFolder,
- '-- Error: could not enter folder ' + depModFolder + '.')
- writeCommand('git checkout ' + versions[depModFolder]['branch'],
- '-- Error: failed to check out branch ' + versions[depModFolder]['branch'] + ' in module ' + depModName + '.')
- writeCommand('git reset --hard ' + versions[depModFolder]['revision'],
- '-- Error: failed to check out commit ' + versions[depModFolder]['revision'] + ' in module ' + depModName + '.')
-
- # write section on application of patches
- def writeApplyPatch(patchRelPath):
- installFile.write('if [ -f ' + patchRelPath + ' ]; then\n')
- writeCommand('git apply ' + patchRelPath,'--Error: failed to apply patch ' + patchRelPath + ' in module ' + depModName + '.', 4)
- installFile.write('else\n')
- installFile.write(' '*4 + exitFunc + ' "--Error: patch ' + patchRelPath + ' was not found."\n')
- installFile.write('fi\n')
-
- if depModFolder in patches and 'unpublished_relpath' in patches[depModFolder]:
- writeApplyPatch(patches[depModFolder]['unpublished_relpath'])
- if depModFolder in patches and 'uncommitted_relpath' in patches[depModFolder]:
- writeApplyPatch(patches[depModFolder]['uncommitted_relpath'])
-
- installFile.write('echo "-- Successfully set up the module ' + depModName + r'\n"' + '\n')
- installFile.write('cd ..\n\n')
-
- # write the module clone first in order for the patches to be present
- if modName in depNames:
- writeCloneModule(modName, modFolder)
- for depModName, depModFolder in zip(depNames, depFolders):
- if depModName != modName:
- writeCloneModule(depModName, depModFolder)
-
- # write configure command
- installFile.write('echo "-- All modules haven been cloned successfully. Configuring project..."\n')
- writeCommand('./dune-common/bin/dunecontrol --opts=' + optsRelPath + ' all', '--Error: could not configure project')
-
- # build the tests of the module
- installFile.write('\n')
- installFile.write('echo "-- Configuring successful. Compiling applications..."\n')
- writeCommand('cd ' + modFolder + "/build-cmake", '--Error: could not enter build directory at ' + modFolder + '/build-cmake')
- writeCommand('make build_tests', '--Error: applications could not be compiled. Please try to compile them manually.')
-
-print("\n-- Successfully created install script file " + instFileName)
-
-if len(patches) > 0:
- print("-> It is recommended that you now commit and publish the 'patches' folder and this install script in your module such that others can use it.")
- print(" IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.")
- print(" That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.")
- print(" If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.")
-
-if modName in depNames: # print gudience to installation if the module is not skipped
- print("\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n")
- print(" ## Installation\n")
- print(" The easiest way of installation is to use the script `" + instFileName + "` provided in this repository.")
- print(" Using `wget`, you can simply install all dependent modules by typing:\n")
- print(" ```sh")
- print(" wget " + versions[modFolder]['remote'] + "/" + instFileName)
- print(" chmod u+x " + instFileName)
- print(" ./" + instFileName)
- print(" ```\n")
-
- if topFolderName: print(" This will create a sub-folder `" + topFolderName + "`, clone all modules into it, configure the entire project and build the applications contained in this module.")
- else: print(" This will clone all modules into the folder from which the script is called, configure the entire project and build the applications contained in this module.")
diff --git a/bin/util/getmoduleinfo.py b/bin/util/moduleinfo.py
similarity index 50%
rename from bin/util/getmoduleinfo.py
rename to bin/util/moduleinfo.py
index 61cb155fe0a19a85ebcf287c1fe42ca8843d7657..ad7a0600eb5265c09ab8f9554f4d83a6cfcb28bd 100644
--- a/bin/util/getmoduleinfo.py
+++ b/bin/util/moduleinfo.py
@@ -1,8 +1,8 @@
import os
-from common import runCommand
-from common import callFromPath
+from util.common import runCommand
+from util.common import callFromPath
+
-# extract information (for the given keys) from a module file
def extractModuleInfos(moduleFile, keys):
results = {}
with open(moduleFile, 'r') as modFile:
@@ -21,48 +21,70 @@ def extractModuleInfos(moduleFile, keys):
return results
-# get info file of a module
+
def getModuleFile(modulePath):
modFile = os.path.join(modulePath, 'dune.module')
if not os.path.exists(modFile):
raise RuntimeError("Could not find module file")
return modFile
-# retrieve single information from a module
+
def getModuleInfo(modulePath, key):
return extractModuleInfos(getModuleFile(modulePath), [key])[key]
-# get the dependencies of a dune module located in the given directory
-def getDependencies(modulePath, verbose=False):
+
+def getDependencies(modulePath, verbose=False, includeSelf=False):
modName = getModuleInfo(modulePath, 'Module')
parentPath = os.path.join(modulePath, '../')
duneControlPath = os.path.join(parentPath, 'dune-common/bin/dunecontrol')
if not os.path.exists(duneControlPath):
- raise RuntimeError('Could not find dunecontrol, expected it to be in {}'.format(duneControlPath))
+ raise RuntimeError(
+ 'Could not find dunecontrol, expected it to be in {}'
+ .format(duneControlPath)
+ )
- run = callFromPath(parentPath)(runCommand)
- for line in run('./dune-common/bin/dunecontrol --module={}'.format(modName)).split('\n'):
+ dcOutput = callFromPath(parentPath)(runCommand)(
+ './dune-common/bin/dunecontrol --module={}'.format(modName)
+ )
+
+ if not dcOutput:
+ raise RuntimeError("Error: call to dunecontrol failed.")
+
+ for line in dcOutput.split('\n'):
if "going to build" in line:
- line = line.replace('going to build', '').replace('---', '').replace('done', '')
+ line = line.replace('going to build', '').strip('-')
line = line.strip('\n').strip()
line = line.split(' ')
deps = line
- # Now we look for the folders with the modules
+ if not includeSelf:
+ deps.remove(modName)
+
if verbose:
- print("Determined the following dependencies: " + ", ".join(deps))
+ print(" -- Determined the following dependencies: " + ", ".join(deps))
+ print(" -- Searching the respective directories...")
result = []
- for dir in [d for d in os.listdir(parentPath) if os.path.isdir(os.path.join(parentPath, d))]:
- try: depModName = getModuleInfo(os.path.join(parentPath, dir), 'Module')
- except: print('--- Note: skipping folder "' + dir + '" as it could not be identifed as dune module')
+ parentFiles = [os.path.join(parentPath, d) for d in os.listdir(parentPath)]
+ for path in filter(os.path.isdir, parentFiles):
+ try:
+ depModName = getModuleInfo(path, 'Module')
+ except Exception:
+ if verbose:
+ print(f" --- skipping folder '{path}' "
+ "as it could not be identifed as dune module")
else:
+ if verbose:
+ print(" --- visited module '{}'".format(depModName))
if depModName in deps:
- result.append({'name': depModName, 'folder': dir})
+ result.append({
+ 'name': depModName,
+ 'folder': os.path.basename(path),
+ 'path': path
+ })
if len(result) != len(deps):
raise RuntimeError("Could not find the folders of all dependencies")
elif verbose:
- print("Found all module folders of the dependencies.")
-
+ print(" -- Found all module folders of the dependencies.")
return result