diff --git a/CHANGELOG.md b/CHANGELOG.md index 209023c65f8c21cf53a10e09978d88a663f2c8c4..3db5ab368b04d2f245591b42c36e4e7efe005f5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +Differences Between DuMu<sup>x</sup> 3.4 and DuMu<sup>x</sup> 3.3 + +### Improvements and Enhancements +- __Several scripts have been translated to Python__: + - `getusedversions.sh` to extract the used Dumux/Dune versions of a module (new script: `bin/util/getusedversions.py`) + - `extractmodulepart.sh` no longer creates an install file, instead, you can now generate install scripts for your module using the new script `bin/util/makeinstallscript.py`. + - Note: the old shells script will be removed after release 3.4. + Differences Between DuMu<sup>x</sup> 3.3 and DuMu<sup>x</sup> 3.2 ============================================= diff --git a/bin/util/common.py b/bin/util/common.py new file mode 100644 index 0000000000000000000000000000000000000000..5dbba0dba23e1921b8979bc621ad845464acd5ac --- /dev/null +++ b/bin/util/common.py @@ -0,0 +1,35 @@ +import os +import sys +import functools +import subprocess + +# execute a command and retrieve the output +def runCommand(command): + try: + return subprocess.run(command, shell=True, check=True, + text=True, capture_output=True).stdout + except Exception as e: + print() + print("An error occurred during subprocess run:") + print("-- command: {}".format(command)) + print("-- folder: {}".format(os.getcwd())) + print("-- error: {}".format(sys.exc_info()[1])) + if "git " in command: + print() + print("It seems that a git command failed. Please check:\n" \ + " -- is the module registered as git repository?\n" \ + " -- is upstream defined for the branch?\n") + raise + +# decorator to call function from within the given path +def callFromPath(path): + def decorator_callFromPath(callFunc): + @functools.wraps(callFunc) + def wrapper_callFromPath(*args, **kwargs): + curPath = os.getcwd() + os.chdir(path) + result = callFunc(*args, **kwargs) + os.chdir(curPath) + return result + return wrapper_callFromPath + return decorator_callFromPath diff --git a/bin/util/extractmodulepart.sh b/bin/util/extractmodulepart.sh index aae0b24ca38d7c311d909ffbcbb54d55d07b1a51..433a79fcd721b96ec76d9cf3d0dd31a3a0b63fdf 100755 --- a/bin/util/extractmodulepart.sh +++ b/bin/util/extractmodulepart.sh @@ -378,38 +378,6 @@ SCRIPT_DIR="${BASH_SOURCE%/*}" if [[ ! -d "$SCRIPT_DIR" ]]; then SCRIPT_DIR="$PWD"; fi -# create install script -touch $MODULE_NAME/install$MODULE_NAME.sh -. "$SCRIPT_DIR/getusedversions.sh" -# execute function from script -echo "Extracting Git status of dependencies and creating patches" -echo "## Dependencies on other DUNE modules" >>$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE -echo "| module | branch | commit hash |" >> $MODULE_NAME/$README_FILE -echo "|:-------|:-------|:------------|" >> $MODULE_NAME/$README_FILE -for MOD in $DEPENDING_MODULE_NAMES -do - if [ $MOD != $MODULE_NAME ]; then - MODULE_DIR=${MOD%*/} - VERSION_OUTPUT=$(getVersionGit $MODULE_DIR $(pwd)/$MODULE_NAME/install$MODULE_NAME.sh) - echo "$VERSION_OUTPUT" - grep "Error:" <<< $VERSION_OUTPUT > /dev/null - EXIT_CODE=$? - if [[ $EXIT_CODE == 0 ]]; then - exit - fi - VERSION_OUTPUT=$(echo "$VERSION_OUTPUT" | tr -s " ") - DEPENDENCY_NAME=$(cut -d " " -f1 <<< $VERSION_OUTPUT) - BRANCH_NAME=$(cut -d " " -f2 <<< $VERSION_OUTPUT) - COMMIT_HASH=$(cut -d " " -f3 <<< $VERSION_OUTPUT) - echo "| $DEPENDENCY_NAME | $BRANCH_NAME | $COMMIT_HASH |" >> $MODULE_NAME/$README_FILE - fi -done - -# move patches folder into module if existing -if [[ -d patches ]]; then - mv patches $MODULE_NAME -fi # output guidence for users echo "" diff --git a/bin/util/getmoduleinfo.py b/bin/util/getmoduleinfo.py new file mode 100644 index 0000000000000000000000000000000000000000..61cb155fe0a19a85ebcf287c1fe42ca8843d7657 --- /dev/null +++ b/bin/util/getmoduleinfo.py @@ -0,0 +1,68 @@ +import os +from common import runCommand +from common import callFromPath + +# extract information (for the given keys) from a module file +def extractModuleInfos(moduleFile, keys): + results = {} + with open(moduleFile, 'r') as modFile: + for line in modFile.readlines(): + line = line.strip('\n').split(':') + if line[0] in keys: + results[line[0]] = line[1].strip() + if len(results) == len(keys): + break + + if len(results) != len(keys): + errMsg = "Could not extract requested information for all keys.\n" + errMsg += "Requested keys: " + ", ".join(keys) + "\n" + errMsg += "Processed keys: " + ", ".join([k for k in results]) + raise RuntimeError(errMsg) + + return results + +# get info file of a module +def getModuleFile(modulePath): + modFile = os.path.join(modulePath, 'dune.module') + if not os.path.exists(modFile): + raise RuntimeError("Could not find module file") + return modFile + +# retrieve single information from a module +def getModuleInfo(modulePath, key): + return extractModuleInfos(getModuleFile(modulePath), [key])[key] + +# get the dependencies of a dune module located in the given directory +def getDependencies(modulePath, verbose=False): + modName = getModuleInfo(modulePath, 'Module') + parentPath = os.path.join(modulePath, '../') + duneControlPath = os.path.join(parentPath, 'dune-common/bin/dunecontrol') + if not os.path.exists(duneControlPath): + raise RuntimeError('Could not find dunecontrol, expected it to be in {}'.format(duneControlPath)) + + run = callFromPath(parentPath)(runCommand) + for line in run('./dune-common/bin/dunecontrol --module={}'.format(modName)).split('\n'): + if "going to build" in line: + line = line.replace('going to build', '').replace('---', '').replace('done', '') + line = line.strip('\n').strip() + line = line.split(' ') + deps = line + + # Now we look for the folders with the modules + if verbose: + print("Determined the following dependencies: " + ", ".join(deps)) + + result = [] + for dir in [d for d in os.listdir(parentPath) if os.path.isdir(os.path.join(parentPath, d))]: + try: depModName = getModuleInfo(os.path.join(parentPath, dir), 'Module') + except: print('--- Note: skipping folder "' + dir + '" as it could not be identifed as dune module') + else: + if depModName in deps: + result.append({'name': depModName, 'folder': dir}) + + if len(result) != len(deps): + raise RuntimeError("Could not find the folders of all dependencies") + elif verbose: + print("Found all module folders of the dependencies.") + + return result diff --git a/bin/util/getusedversions.py b/bin/util/getusedversions.py new file mode 100755 index 0000000000000000000000000000000000000000..18b407051a3695fbf3cd86b9059f1ce2dcdb5e1e --- /dev/null +++ b/bin/util/getusedversions.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import os +import argparse +from common import runCommand +from common import callFromPath + +# print warning message for scanned folders that are not git repositories +def printNoGitRepoWarning(folderPath): + print("Folder " + folderPath + " does not seem to be the top level" \ + "of a git repository and will be skipped. Make sure not to call " \ + "this script from a sub-directory of a git repository.") + +# raise error due to untracked files present in the given module folder +def raiseUntrackedFilesError(folderPath): + raise RuntimeError('Found untracked files in module folder: "' + folderPath + '". ' \ + 'Please commit, stash, or remove them.') + +# returns true if the given folder is a git repository +def isGitRepository(modFolderPath): + return os.path.exists(os.path.join(modFolderPath, '.git')) + +# returns true if a module contains untracked files +def hasUntrackedFiles(modFolderPath): + run = callFromPath(modFolderPath)(runCommand) + return run('git ls-files --others --exclude-standard') != '' + +# function to extract git version information for modules +# returns a dictionary containing module information for each given module folder +def getUsedVersions(modFolderPaths, ignoreUntracked=False): + + result = {} + for modFolderPath in modFolderPaths: + + # make sure this is the top level of a git repository + if not isGitRepository(modFolderPath): + printNoGitRepoWarning(modFolderPath) + else: + if not ignoreUntracked and hasUntrackedFiles(modFolderPath): + raiseUntrackedFilesError(modFolderPath) + + run = callFromPath(modFolderPath)(runCommand) + result[modFolderPath] = {} + result[modFolderPath]['remote'] = run('git ls-remote --get-url').strip('\n') + result[modFolderPath]['revision'] = run('git log -n 1 --format=%H @{upstream}').strip('\n') + result[modFolderPath]['date'] = run('git log -n 1 --format=%ai @{upstream}').strip('\n') + result[modFolderPath]['author'] = run('git log -n 1 --format=%an @{upstream}').strip('\n') + result[modFolderPath]['branch'] = run('git rev-parse --abbrev-ref HEAD').strip('\n') + + return result + +# create patches for unpublished commits and uncommitted changes in modules +def getPatches(modFolderPaths, ignoreUntracked=False): + + result = {} + for modFolderPath in modFolderPaths: + + # make sure this is the top level of a git repository + if not isGitRepository(modFolderPath): + printNoGitRepoWarning(modFolderPath) + else: + if not ignoreUntracked and hasUntrackedFiles(modFolderPath): + raiseUntrackedFilesError(modFolderPath) + + run = callFromPath(modFolderPath)(runCommand) + unpubPatch = run('git format-patch --stdout @{upstream}') + unCommPatch = run('git diff') + if unpubPatch != '' or unCommPatch != '': result[modFolderPath] = {} + if unpubPatch != '': result[modFolderPath]['unpublished'] = unpubPatch + if unCommPatch != '': result[modFolderPath]['uncommitted'] = unCommPatch + + return result + +# prints the detected versions as table +def printVersionTable(versions): + print("\t| {:^50} | {:^50} | {:^50} | {:^30} |".format('module folder', 'branch', 'commit hash', 'commit date')) + print("\t" + 193*'-') + for folder, versionInfo in versions.items(): + print("\t| {:^50} | {:^50} | {:^50} | {:^30} |".format(folder, versionInfo['branch'], versionInfo['revision'], versionInfo['date'])) + + +# For standalone execution +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='This script extracts the used dune/dumux versions.') + parser.add_argument('-p', '--path', required=False, help='the path to the top folder containing your dune/dumux modules') + parser.add_argument('-i', '--ignoreuntracked', required=False, action='store_true', help='use this flag to ignore untracked files present in the modules') + parser.add_argument('-s', '--skipfolders', required=False, nargs='*', help='a list of module folders to be skipped') + cmdArgs = vars(parser.parse_args()) + + modulesPath = os.getcwd() if not cmdArgs['path'] else os.path.join(os.getcwd(), cmdArgs['path']) + print('\nDetermining the versions of all dune modules in the folder: ' + modulesPath) + + def getPath(modFolder): + return os.path.join(modulesPath, modFolder) + + modFolderPaths = [getPath(dir) for dir in os.listdir(modulesPath) if os.path.isdir(getPath(dir))] + if cmdArgs['skipfolders']: + cmdArgs['skipfolders'] = [f.strip('/') for f in cmdArgs['skipfolders']] + modFolderPaths = [d for d in modFolderPaths if os.path.basename(d.strip('/')) not in cmdArgs['skipfolders']] + versions = getUsedVersions(modFolderPaths, True) + + print("\nDetected the following versions:") + printVersionTable(versions) + + # maybe check untracked files + if not cmdArgs['ignoreuntracked']: + modsWithUntracked = [f for f in versions if hasUntrackedFiles(f)] + if modsWithUntracked: + print('\n') + print('#'*56) + print('WARNING: Found untracked files in the following modules:\n\n') + print('\n'.join(modsWithUntracked)) + print('\nPlease make sure that these are not required for your purposes.') + print('If not, you can run this script with the option -i/--ignoreuntracked to suppress this warning.') + print('#'*56) diff --git a/bin/util/getusedversions.sh b/bin/util/getusedversions.sh index 102e2f0eed1d3eb3c00f69702232f9736162d1af..c1e23cd1145198840d38f723b267622ab33e129d 100755 --- a/bin/util/getusedversions.sh +++ b/bin/util/getusedversions.sh @@ -6,9 +6,11 @@ # (c) 2016 Thomas Fetzer # (c) 2016 Christoph Grüninger # +# NOTE: This script is deprecated and will be removed after release 3.4. +# Please use the corresponding Python script "getusedversions.py" if [ "$1" = "-h" ]; then - echo "USAGE: ./getDumuxDuneVersions.sh" + echo "USAGE: ./getusedversions.sh" echo; exit fi @@ -86,6 +88,7 @@ function getVersionGit # run script from command line # suppressed for use of external script when variable set accordingly if [ "$CALL_FROM_EXTERNAL_SCRIPT" != "yes" ]; then + echo "NOTE: This script is deprecated, please use getusedversions.py instead!" echo "# DUNE/DUMUX VERSIONS" > $OUTFILE echo "Creating file containing the version numbers:" diff --git a/bin/util/makeinstallscript.py b/bin/util/makeinstallscript.py new file mode 100755 index 0000000000000000000000000000000000000000..f91e53f25683edf4337859db8e54639468286eec --- /dev/null +++ b/bin/util/makeinstallscript.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 + +# script to generate an install script for a dune-module, +# accounting for non-published commits and local changes + +import os +import sys +import argparse +import subprocess +from getmoduleinfo import * +from getusedversions import * + +if sys.version_info[0] < 3: + sys.exit("\nError': Python3 required") + + +################### +# parse arguments +parser = argparse.ArgumentParser( + description='This script generates an install script for your dune module,' \ + 'taking into account non-published commits and local changes.\n' \ + 'This expects that your module is a git repository and that a ' \ + 'remote origin exists and has been set already.' +) +parser.add_argument('-p', '--path', required=True, help='The path to your dune module') +parser.add_argument('-f', '--filename', required=False, help='Name of the file in which to write the install script') +parser.add_argument('-i', '--ignoreuntracked', required=False, action='store_true', help='Use this flag to ignore untracked files present in the modules') +parser.add_argument('-t', '--topfoldername', required=False, default='DUMUX', + help='Name of the folder that the install script creates upon execution to install the modules in. '\ + 'If you pass an empty string, no folder will be created and installation happens in place.') +parser.add_argument('-o', '--optsfile', required=False, + help='Provide custom opts file to be used for the call to dunecontrol. '\ + 'Note that this file is required to be contained and committed within the module or its dependencies.') +cmdArgs = vars(parser.parse_args()) + + +#################### +# Welcome message +modPath = os.path.abspath( os.path.join(os.getcwd(), cmdArgs['path']) ) +modParentPath = os.path.abspath( os.path.join(modPath, '../') ) +modFolder = os.path.relpath(modPath, modParentPath) + +try: modName = getModuleInfo(modPath, 'Module') +except: + sys.exit("\Error: Could not determine module name. Make sure the path to\n" \ + " your module is correct and that a dune.module file is present.") + +instFileName = 'install_' + modName + '.sh' if not cmdArgs['filename'] else cmdArgs['filename'] +print("\n-- Creating install script '{}' for the module '{}' in the folder '{}'".format(instFileName, modName, modPath)) + + +################### +# get dependencies +print("\n-- Determining the dependencies") +deps = getDependencies(modPath) +if len(deps) > 0: + print("-> Found the following dependencies") + print("\t| {:^50} | {:^50} |".format('module name', 'module folder')) + print("\t" + 107*'-') + for dep in deps: print('\t| {:^50} | {:^50} |'.format(dep['name'], dep['folder'])) +else: + sys.exit("Error: Could not determine dependencies. At least the module itself should appear.") + +depNames = [dep['name'] for dep in deps] +depFolders = [dep['folder'] for dep in deps] +depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] + + +################################# +# determine specific commits of all modules +print("\n-- Determining the module versions") +try: versions = getUsedVersions(depFolderPaths, cmdArgs['ignoreuntracked']) +except Exception as err: + print('\nCaught exception: ' + str(err)) + if 'untracked' in str(err): + print('If you are certain that the untracked files are not needed for the ' \ + 'installation of the modules, run this script with the -i flag.') + sys.exit(1) + +if len(versions) != len(depNames): + sys.exit("Error': Could not determine versions of all modules") + +print("-> Detected the following versions") +printVersionTable(versions) + +################################# +# create patches if necessary +print("\n-- Creating patches for unpublished commits and uncommitted changes") +patches = getPatches(depFolderPaths, cmdArgs['ignoreuntracked']) + +# create patch files +if len(patches) > 0: + patchesPath = os.path.join(modPath, 'patches') + if not os.path.exists(patchesPath): + os.mkdir(patchesPath) + print("-> Created a folder 'patches' in your module. You should commit this to your\n"\ + " repository in order for the installation script to work on other machines.") + else: + print("-> Adding patches to the 'patches' folder in your module. Make sure to commit\n"\ + " the newly added patches in order for the install script to work on other machines.") + + # get a non-used patch file name (makes sure not to overwrite anything) + def getPatchFileName(depModPath, targetName): + i = 1 + fileName = os.path.join(patchesPath, targetName + '.patch') + while os.path.exists(fileName): + fileName = os.path.join(patchesPath, targetName + '_' + str(i) + '.patch') + i += 1 + + return fileName, os.path.relpath(fileName, depModPath) + + # function to write a new patch file (returns the path to the new file) + def writeDepModPatchFile(depModPath, depModName, type): + patchPath, patchRelPath = getPatchFileName(depModPath, depModName + '_' + type) + patches[depModPath][type + '_relpath'] = patchRelPath + open(patchPath, 'w').write(patches[depModPath][type]) + return patchPath + + print("-> Created patch files:") + for depModPath in patches.keys(): + depModName = getModuleInfo(depModPath, "Module") + if 'unpublished' in patches[depModPath]: + print(' '*3 + writeDepModPatchFile(depModPath, depModName, 'unpublished')) + if 'uncommitted' in patches[depModPath]: + print(' '*3 + writeDepModPatchFile(depModPath, depModName, 'uncommitted')) + +else: + print("-> No Patches required") + + +################################## +# write installation shell script + +# in the install script, we work with relative paths from the parent to module folder +def getModRelPath(path): return os.path.relpath(path, modParentPath) +versions = {getModRelPath(modPath): val for modPath, val in versions.items()} +patches = {getModRelPath(modPath): val for modPath, val in patches.items()} + +topFolderName = cmdArgs['topfoldername'] +optsRelPath = 'dumux/cmake.opts' +if cmdArgs['optsfile']: + optsPath = os.path.abspath( os.path.join(os.getcwd(), cmdArgs['optsfile']) ) + optsRelPath = os.path.relpath(optsPath, modParentPath) + +# TODO: add support for different install script languages (e.g. Python). +with open(instFileName, 'w') as installFile: + installFile.write('#!/bin/bash\n\n') + installFile.write('#'*80 + '\n') + installFile.write('# This script installs the module "' + modName + '" together with all dependencies.\n') + installFile.write('\n') + + exitFunc = 'exitWith' + installFile.write('# defines a function to exit with error message\n') + installFile.write(exitFunc + ' ()\n') + installFile.write('{\n') + installFile.write(' echo ' + r'"\n$1"' +'\n') + installFile.write(' exit 1\n') + installFile.write('}\n\n') + + # function to write a command with error check into the script file + def writeCommand(command, errorMessage, indentationLevel = 0): + installFile.write(' '*indentationLevel + 'if ! ' + command + ';') + installFile.write(' then ' + exitFunc + ' "' + errorMessage + '"; fi\n') + + # write section on creating the top folder + if topFolderName: + installFile.write('# Everything will be installed into a newly created sub-folder named "' + topFolderName + '".\n') + installFile.write('echo "Creating the folder ' + topFolderName + ' to install the modules in"\n') + writeCommand('mkdir -p ' + topFolderName , '--Error: could not create top folder ' + topFolderName) + writeCommand('cd ' + topFolderName, '--Error: could not enter top folder ' + topFolderName) + installFile.write('\n') + else: + installFile.write('# Everything will be installed inside the folder from which the script is executed.\n\n') + + # function to write installation procedure for a module + def writeCloneModule(depModName, depModFolder): + installFile.write('# ' + depModName + '\n') + installFile.write('# ' + versions[depModFolder]['branch'] + ' # ' + + versions[depModFolder]['revision'] + ' # ' + + versions[depModFolder]['date'] + ' # ' + + versions[depModFolder]['author'] + '\n') + + writeCommand('git clone ' + versions[depModFolder]['remote'], + '-- Error: failed to clone ' + depModName + '.') + writeCommand('cd ' + depModFolder, + '-- Error: could not enter folder ' + depModFolder + '.') + writeCommand('git checkout ' + versions[depModFolder]['branch'], + '-- Error: failed to check out branch ' + versions[depModFolder]['branch'] + ' in module ' + depModName + '.') + writeCommand('git reset --hard ' + versions[depModFolder]['revision'], + '-- Error: failed to check out commit ' + versions[depModFolder]['revision'] + ' in module ' + depModName + '.') + + # write section on application of patches + def writeApplyPatch(patchRelPath): + installFile.write('if [ -f ' + patchRelPath + ' ]; then\n') + writeCommand('git apply ' + patchRelPath,'--Error: failed to apply patch ' + patchRelPath + ' in module ' + depModName + '.', 4) + installFile.write('else\n') + installFile.write(' '*4 + exitFunc + ' "--Error: patch ' + patchRelPath + ' was not found."\n') + installFile.write('fi\n') + + if depModFolder in patches and 'unpublished_relpath' in patches[depModFolder]: + writeApplyPatch(patches[depModFolder]['unpublished_relpath']) + if depModFolder in patches and 'uncommitted_relpath' in patches[depModFolder]: + writeApplyPatch(patches[depModFolder]['uncommitted_relpath']) + + installFile.write('echo "-- Successfully set up the module ' + depModName + r'\n"' + '\n') + installFile.write('cd ..\n\n') + + # write the module clone first in order for the patches to be present + writeCloneModule(modName, modFolder) + for depModName, depModFolder in zip(depNames, depFolders): + if depModName != modName: + writeCloneModule(depModName, depModFolder) + + # write configure command + installFile.write('echo "-- All modules haven been cloned successfully. Configuring project..."\n') + writeCommand('./dune-common/bin/dunecontrol --opts=' + optsRelPath + ' all', '--Error: could not configure project') + + # build the tests of the module + installFile.write('\n') + installFile.write('echo "-- Configuring successful. Compiling applications..."\n') + writeCommand('cd ' + modFolder + "/build-cmake", '--Error: could not enter build directory at ' + modFolder + '/build-cmake') + writeCommand('make build_tests', '--Error: applications could not be compiled. Please try to compile them manually.') + +print("\n-- Successfully created install script file " + instFileName) +if len(patches) > 0: + print("-> It is recommended that you now commit and publish the 'patches' folder and this install script in your module such that others can use it.") + print(" IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.") + print(" That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.") + print(" If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") + +print("\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n") +print(" ## Installation\n") +print(" The easiest way of installation is to use the script `" + instFileName + "` provided in this repository.") +print(" Using `wget`, you can simply install all dependent modules by typing:\n") +print(" ```sh") +print(" wget " + versions[modFolder]['remote'] + "/" + instFileName) +print(" chmod u+x " + instFileName) +print(" ./" + instFileName) +print(" ```\n") + +if topFolderName: print(" This will create a sub-folder `" + topFolderName + "`, clone all modules into it, configure the entire project and build the applications contained in this module.") +else: print(" This will clone all modules into the folder from which the script is called, configure the entire project and build the applications contained in this module.")