From b6851bfdb975e43a7e50632ab6861bd8c5883729 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Thu, 27 May 2021 16:03:50 +0000 Subject: [PATCH 01/47] [getusedversions] Return last commit that is synced with the master branch --- bin/util/getusedversions.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/util/getusedversions.py b/bin/util/getusedversions.py index 18b407051a..a9c933755b 100755 --- a/bin/util/getusedversions.py +++ b/bin/util/getusedversions.py @@ -20,11 +20,28 @@ def raiseUntrackedFilesError(folderPath): def isGitRepository(modFolderPath): return os.path.exists(os.path.join(modFolderPath, '.git')) + # returns true if a module contains untracked files def hasUntrackedFiles(modFolderPath): run = callFromPath(modFolderPath)(runCommand) return run('git ls-files --others --exclude-standard') != '' + +# get the most recent commit that also exists on the remote master/release branch +# maybe used to find a commit we can use as basis for a pub module +def mostRecentCommonCommitWithRemote(modFolderPath, match=['/master', '/release']): + run = callFromPath(modFolderPath)(runCommand) + revList = run('git rev-list HEAD').split('\n') + for rev in revList: + remoteBranches = run('git branch -r --contains {}'.format(rev)) + for m in match: + if m in remoteBranches: + return rev + + raise RuntimeError('Could not suitable find ancestor commit' + ' that is synced with master on remote') + + # function to extract git version information for modules # returns a dictionary containing module information for each given module folder def getUsedVersions(modFolderPaths, ignoreUntracked=False): @@ -42,9 +59,13 @@ def getUsedVersions(modFolderPaths, ignoreUntracked=False): run = callFromPath(modFolderPath)(runCommand) result[modFolderPath] = {} result[modFolderPath]['remote'] = run('git ls-remote --get-url').strip('\n') - result[modFolderPath]['revision'] = run('git log -n 1 --format=%H @{upstream}').strip('\n') - result[modFolderPath]['date'] = run('git log -n 1 --format=%ai @{upstream}').strip('\n') - result[modFolderPath]['author'] = run('git log -n 1 --format=%an @{upstream}').strip('\n') + # update remote to make sure we find all upstream commits + run('git fetch {}'.format(result[modFolderPath]['remote'])) + rev = mostRecentCommonCommitWithRemote(modFolderPaths) + result[modFolderPath]['revision'] = rev + result[modFolderPath]['date'] = run('git log -n 1 --format=%ai {}'.format(rev)).strip('\n') + result[modFolderPath]['author'] = run('git log -n 1 --format=%an {}'.format(rev)).strip('\n') + # this may return HEAD if we are on some detached HEAD tree result[modFolderPath]['branch'] = run('git rev-parse --abbrev-ref HEAD').strip('\n') return result -- GitLab From 29b40274ac6033a020f052fee6b698b2fe8510be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= Date: Mon, 31 May 2021 19:35:48 +0200 Subject: [PATCH 02/47] [bin][extractmod] add utility functions --- bin/extractmodule/util.py | 135 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 bin/extractmodule/util.py diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py new file mode 100644 index 0000000000..1cf2da7a5d --- /dev/null +++ b/bin/extractmodule/util.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import os +import sys + +try: + path = os.path.split(os.path.abspath(__file__))[0] + sys.path.append(os.path.join(path, '../util')) + + from common import callFromPath, runCommand +except Exception: + sys.exit('Could not import common module') + + +def isGitRepository(pathToRepo='.'): + try: + run = callFromPath(pathToRepo)(runCommand) + run('git status') + return True + except Exception: + return False + + +def getRemote(pathToRepo='.'): + run = callFromPath(pathToRepo)(runCommand) + return run('git ls-remote --get-url').strip('\n') + + +def fetchRepo(remote, pathToRepo='.'): + run = callFromPath(pathToRepo)(runCommand) + run('git fetch {}'.format(remote)) + + +def hasUntrackedFiles(pathToRepo='.'): + run = callFromPath(pathToRepo)(runCommand) + return run('git ls-files --others --exclude-standard') != '' + + +def isPersistentBranch(branchName): + if branchName == 'origin/master': + return True + if branchName.startswith('origin/releases/'): + return True + return False + + +# get the most recent commit that also exists on remote master/release branch +# may be used to find a commit we can use as basis for a pub module +def mostRecentCommonCommitWithRemote(modFolderPath, + branchFilter=isPersistentBranch): + run = callFromPath(modFolderPath)(runCommand) + + def findBranches(sha): + candidates = run('git branch -r --contains {}'.format(sha)).split('\n') + candidates = [branch.strip().split(' ->')[0] for branch in candidates] + return list(filter(branchFilter, candidates)) + + revList = run('git rev-list HEAD').split('\n') + for rev in revList: + branches = findBranches(rev) + if branches: + return branches[0] + + raise RuntimeError('Could not find suitable ancestor commit' + ' on a branch that matches the given filter') + + +# function to extract persistent, remotely available git versions for all +def getPersistentVersions(modFolderPaths, ignoreUntracked=False): + + result = {} + for modFolderPath in modFolderPaths: + + if not isGitRepository(modFolderPath): + raise Exception('Folder is not a git repository') + + if hasUntrackedFiles(modFolderPath) and not ignoreUntracked: + raise Exception("Found untracked files in '{}'." + "Please commit, stash, or remove them." + .format(modFolderPath)) + + result[modFolderPath] = {} + result[modFolderPath]['remote'] = getRemote(modFolderPath) + + # update remote to make sure we find all upstream commits + fetchRepo(result[modFolderPath]['remote'], modFolderPath) + + rev = mostRecentCommonCommitWithRemote(modFolderPath) + run = callFromPath(modFolderPath)(runCommand) + + result[modFolderPath]['revision'] = rev + result[modFolderPath]['date'] = run( + 'git log -n 1 --format=%ai {}'.format(rev) + ).strip('\n') + result[modFolderPath]['author'] = run( + 'git log -n 1 --format=%an {}'.format(rev) + ).strip('\n') + + # this may return HEAD if we are on some detached HEAD tree + result[modFolderPath]['branch'] = run( + 'git rev-parse --abbrev-ref HEAD' + ).strip('\n') + + return result + + +def getPatches(persistentVersions): + result = {} + for path, gitInfo in persistentVersions.items(): + run = callFromPath(path)(runCommand) + + unCommPatch = run('git diff') + unpubPatch = run( + 'git format-patch --stdout {}'.format(gitInfo['revision']) + ) + + if unpubPatch or unCommPatch: + result[path] = {} + if unpubPatch: + result[path]['unpublished'] = unpubPatch + if unCommPatch: + result[path]['uncommitted'] = unCommPatch + return result + + +def printVersionTable(versions): + print("\t| {:^50} | {:^50} | {:^50} | {:^30} |" + .format('module folder', 'branch', 'commit hash', 'commit date')) + print("\t" + 193*'-') + for folder, versionInfo in versions.items(): + print("\t| {:^50} | {:^50} | {:^50} | {:^30} |" + .format(folder, + versionInfo['branch'], + versionInfo['revision'], + versionInfo['date'])) -- GitLab From 0d760ae29e838ac3fb6ebd580e8b01b20ea9173e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= Date: Mon, 31 May 2021 19:36:37 +0200 Subject: [PATCH 03/47] [bin][util] move and adjust install script --- bin/extractmodule/makeinstallscript.py | 323 +++++++++++++++++++++++++ bin/util/makeinstallscript.py | 250 ------------------- 2 files changed, 323 insertions(+), 250 deletions(-) create mode 100644 bin/extractmodule/makeinstallscript.py delete mode 100755 bin/util/makeinstallscript.py diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py new file mode 100644 index 0000000000..681f36e7fe --- /dev/null +++ b/bin/extractmodule/makeinstallscript.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 + +# script to generate an install script for a dune-module, +# accounting for non-published commits and local changes + +import os +import sys +import argparse + +from util import getPersistentVersions +from util import printVersionTable +from util import getPatches + +try: + path = os.path.split(os.path.abspath(__file__))[0] + sys.path.append(os.path.join(path, '../bin/util')) + + from getmoduleinfo import getModuleInfo, getDependencies +except Exception: + sys.exit('Could not import getModuleInfo') + +if sys.version_info[0] < 3: + sys.exit("\nError': Python3 required") + + +################### +# parse arguments +parser = argparse.ArgumentParser( + description='This script generates an install script for your dune module,' + 'taking into account non-published commits & local changes.\n' + 'This expects that your module is a git repository and that a ' + 'remote origin exists and has been set already.' +) +parser.add_argument('-p', '--path', + required=True, + help='The path to your dune module') +parser.add_argument('-f', '--filename', + required=False, + help='File in which to write the install script') +parser.add_argument('-i', '--ignoreuntracked', + required=False, action='store_true', + help='Use this to ignore untracked files present') +parser.add_argument('-t', '--topfoldername', + required=False, default='DUMUX', + help='Name of the folder that the install script creates ' + 'upon execution to install the modules in. If you ' + 'pass an empty string, no folder will be created ' + 'and installation happens in place.') +parser.add_argument('-o', '--optsfile', + required=False, + help='Provide custom opts file to be used for the call to ' + 'dunecontrol. Note that this file is required to be ' + 'contained and committed within the module or its ' + 'dependencies.') +cmdArgs = vars(parser.parse_args()) + + +cwd = os.getcwd() +modPath = os.path.abspath(os.path.join(cwd, cmdArgs['path'])) +modParentPath = os.path.abspath(os.path.join(modPath, '../')) +modFolder = os.path.relpath(modPath, modParentPath) + +try: + modName = getModuleInfo(modPath, 'Module') +except Exception: + sys.exit("\nError: Could not determine module name. Make sure the module\n" + " path is correct and that it contains 'dune.module'.") + +if not cmdArgs['filename']: + instFileName = 'install_' + modName + '.sh' +else: + instFileName = cmdArgs['filename'] +print("\n-- Creating install script '{}' for module '{}' in folder '{}'" + .format(instFileName, modName, modPath)) + + +print("\n-- Determining the dependencies") +deps = getDependencies(modPath) +if len(deps) > 0: + print("-> Found the following dependencies") + print("\t| {:^50} | {:^50} |".format('module name', 'module folder')) + print("\t" + 107*'-') + for dep in deps: + print('\t| {:^50} | {:^50} |'.format(dep['name'], dep['folder'])) +else: + sys.exit("Error: Could not determine module dependencies.") + +depNames = [dep['name'] for dep in deps] +depFolders = [dep['folder'] for dep in deps] +depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] + + +print("\n-- Determining the module versions") +try: + ignoreUntracked = cmdArgs['ignoreuntracked'] + versions = getPersistentVersions(depFolderPaths, ignoreUntracked) +except Exception as err: + print('\nCaught exception: ' + str(err)) + if 'untracked' in str(err): + print('If you are certain that the untracked files are not needed for ' + 'the module installation, run this script with the -i flag.') + sys.exit(1) + +if len(versions) != len(depNames): + sys.exit("Error': Could not determine versions of all modules") + +print("-> The following (remotely available) versions are used as basis") +print(" on top of which we will generate the required patches") +printVersionTable(versions) + + +print("\n-- Creating patches for unpublished commits and uncommitted changes") +patches = getPatches(versions) + +# create patch files +if len(patches) > 0: + print("-> Placing patches in the folder 'patches' in your module. You " + "should commit them to your repository in order for the install" + "script to work on other machines.") + patchesPath = os.path.join(modPath, 'patches') + os.makedirs(patchesPath, exist_ok=True) + + def getPatchFileNameAndRelPath(depModPath, targetName): + i = 1 + fileName = os.path.join(patchesPath, targetName + '.patch') + while os.path.exists(fileName): + fileName = os.path.join(patchesPath, + targetName + '_' + str(i) + '.patch') + i += 1 + + return fileName, os.path.relpath(fileName, depModPath) + + def writePatch(depModPath, depModName, type): + patchName = depModName + '_' + type + pPath, pRelPath = getPatchFileNameAndRelPath(depModPath, patchName) + patches[depModPath][type + '_relpath'] = pRelPath + open(pPath, 'w').write(patches[depModPath][type]) + return pPath + + print("-> Created patch files:") + for depModPath in patches.keys(): + depModName = getModuleInfo(depModPath, "Module") + if 'unpublished' in patches[depModPath]: + print(' '*3 + writePatch(depModPath, depModName, 'unpublished')) + if 'uncommitted' in patches[depModPath]: + print(' '*3 + writePatch(depModPath, depModName, 'uncommitted')) + +else: + print("-> No Patches required") + + +# write installation shell script (switch to relative paths) +versions = {os.path.relpath(p, modParentPath): v for p, v in versions.items()} +patches = {os.path.relpath(p, modParentPath): v for p, v in patches.items()} + +topFolderName = cmdArgs['topfoldername'] +optsRelPath = 'dumux/cmake.opts' +if cmdArgs['optsfile']: + optsPath = os.path.abspath(os.path.join(os.getcwd(), cmdArgs['optsfile'])) + optsRelPath = os.path.relpath(optsPath, modParentPath) + +# TODO: add support for different install script languages (e.g. Python). +with open(instFileName, 'w') as installFile: + installFile.write("#!/bin/bash\n\n") + installFile.write("#"*80 + "\n") + installFile.write("# This script installs the module '{}'" + " together with all dependencies.\n" + "\n".format(modName)) + + exitFunc = 'exitWith' + installFile.write( + '# defines a function to exit with error message\n' + '{} ()\n'.format(exitFunc) + + + '{\n' + ' echo "\\n$1"\n' + ' exit 1\n' + '}\n\n' + ) + + unitIndentation = ' '*4 + + def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): + indent = unitIndentation*indentationLevel + nextIndent = indent + unitIndentation + installFile.write( + f"{indent}if ! {command}; then\n" + f"{nextIndent}echo \"{errorMessage}\"\n" + "fi\n" + ) + + if topFolderName: + installFile.write( + '# Everything will be installed into a new folder "{}".\n' + .format(topFolderName) + ) + installFile.write( + 'echo "Creating the folder {} to install the modules in"\n' + .format(topFolderName) + + ) + + writeCommandWithErrorCheck( + 'mkdir -p {}'.format(topFolderName), + '--Error: could not create top folder {}'.format(topFolderName) + ) + + writeCommandWithErrorCheck( + 'cd {}'.format(topFolderName), + '--Error: could not enter top folder {}'.format(topFolderName)) + + installFile.write('\n') + else: + installFile.write( + '# Everything will be installed inside the folder from which the' + ' script is executed.\n\n' + ) + + def writeCloneModule(depModName, depModFolder): + installFile.write('# ' + depModName + '\n') + installFile.write('# ' + versions[depModFolder]['branch'] + ' # ' + + versions[depModFolder]['revision'] + ' # ' + + versions[depModFolder]['date'] + ' # ' + + versions[depModFolder]['author'] + '\n') + + writeCommandWithErrorCheck( + 'git clone {}'.format(versions[depModFolder]['remote']), + '-- Error: failed to clone {}.'.format(depModName) + ) + writeCommandWithErrorCheck( + 'cd {}'.format(depModFolder), + '-- Error: could not enter folder {}.'.format(depModFolder) + ) + writeCommandWithErrorCheck( + 'git checkout {}'.format(versions[depModFolder]['branch']), + '-- Error: failed to check out branch {} in module {}.' + .format(versions[depModFolder]['branch'], depModName) + ) + writeCommandWithErrorCheck( + 'git reset --hard {}'.format(versions[depModFolder]['revision']), + '-- Error: failed to check out commit {} in module {}.' + .format(versions[depModFolder]['revision'], depModName) + ) + + # write section on application of patches + def writeApplyPatch(patchRelPath): + patchError = '--Error: patch {} was not found'.format(patchRelPath) + installFile.write('if [ -f {} ]; then\n'.format(patchRelPath)) + writeCommandWithErrorCheck( + 'git apply {}'.format(patchRelPath), + '--Error: failed to apply patch {} in module {}' + .format(patchRelPath, depModName), + indentationLevel=1 + ) + installFile.write("else\n" + f"{unitIndentation}{exitFunc} " + f"\"{patchError}\".\n") + installFile.write('fi\n') + + if depModFolder in patches: + if'unpublished_relpath' in patches[depModFolder]: + writeApplyPatch(patches[depModFolder]['unpublished_relpath']) + if 'uncommitted_relpath' in patches[depModFolder]: + writeApplyPatch(patches[depModFolder]['uncommitted_relpath']) + + installFile.write('echo "-- Successfully set up the module {} \\n"\n' + .format(depModName) + '\n') + installFile.write('cd ..\n\n') + + # write the module clone first in order for the patches to be present + if modName in depNames: + writeCloneModule(modName, modFolder) + for depModName, depModFolder in zip(depNames, depFolders): + if depModName != modName: + writeCloneModule(depModName, depModFolder) + + # write configure command + installFile.write('echo "-- All modules haven been cloned successfully. ' + 'Configuring project..."\n') + writeCommandWithErrorCheck( + './dune-common/bin/dunecontrol --opts={} all'.format(optsRelPath), + '--Error: could not configure project' + ) + + # build the tests of the module + installFile.write( + '\n' + 'echo "-- Configuring successful. Compiling applications..."\n' + ) + writeCommandWithErrorCheck( + 'cd {}/build-cmake'.format(modFolder), + '--Error: could not enter build directory at {}/build-cmake' + .format(modFolder) + ) + writeCommandWithErrorCheck( + 'make build_tests', + '--Error: applications could not be compiled. ' + 'Please try to compile them manually.' + ) + +print("\n-- Successfully created install script file " + instFileName) + +if len(patches) > 0: + print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" + " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" + " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" + " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") + +print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" + f" ## Installation\n" + f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" + f" Using `wget`, you can simply install all dependent modules by typing:\n" + f"\n" + f" ```sh\n" + f" wget {versions[modFolder]['remote']}/{instFileName}\n" + f" chmod u+x {instFileName}\n" + f" ./{instFileName}\n" + f" ```\n") + +if topFolderName: + print(f" This will create a sub-folder `{topFolderName}`, clone all modules into it, configure the entire project and build the applications contained in this module.") +else: + print(" This will clone all modules into the folder from which the script is called, configure the entire project and build the applications contained in this module.") diff --git a/bin/util/makeinstallscript.py b/bin/util/makeinstallscript.py deleted file mode 100755 index 8324859005..0000000000 --- a/bin/util/makeinstallscript.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env python3 - -# script to generate an install script for a dune-module, -# accounting for non-published commits and local changes - -import os -import sys -import argparse -import subprocess -from getmoduleinfo import * -from getusedversions import * - -if sys.version_info[0] < 3: - sys.exit("\nError': Python3 required") - - -################### -# parse arguments -parser = argparse.ArgumentParser( - description='This script generates an install script for your dune module,' \ - 'taking into account non-published commits and local changes.\n' \ - 'This expects that your module is a git repository and that a ' \ - 'remote origin exists and has been set already.' -) -parser.add_argument('-p', '--path', required=True, help='The path to your dune module') -parser.add_argument('-f', '--filename', required=False, help='Name of the file in which to write the install script') -parser.add_argument('-i', '--ignoreuntracked', required=False, action='store_true', help='Use this flag to ignore untracked files present in the modules') -parser.add_argument('-t', '--topfoldername', required=False, default='DUMUX', - help='Name of the folder that the install script creates upon execution to install the modules in. '\ - 'If you pass an empty string, no folder will be created and installation happens in place.') -parser.add_argument('-o', '--optsfile', required=False, - help='Provide custom opts file to be used for the call to dunecontrol. '\ - 'Note that this file is required to be contained and committed within the module or its dependencies.') -parser.add_argument('-s', '--skipfolders', required=False, nargs='*', help='a list of module folders to be skipped') -cmdArgs = vars(parser.parse_args()) - - -#################### -# Welcome message -modPath = os.path.abspath( os.path.join(os.getcwd(), cmdArgs['path']) ) -modParentPath = os.path.abspath( os.path.join(modPath, '../') ) -modFolder = os.path.relpath(modPath, modParentPath) - -try: modName = getModuleInfo(modPath, 'Module') -except: - sys.exit("\Error: Could not determine module name. Make sure the path to\n" \ - " your module is correct and that a dune.module file is present.") - -instFileName = 'install_' + modName + '.sh' if not cmdArgs['filename'] else cmdArgs['filename'] -print("\n-- Creating install script '{}' for the module '{}' in the folder '{}'".format(instFileName, modName, modPath)) - - -################### -# get dependencies -print("\n-- Determining the dependencies") -deps = getDependencies(modPath) -if len(deps) > 0: - print("-> Found the following dependencies") - print("\t| {:^50} | {:^50} |".format('module name', 'module folder')) - print("\t" + 107*'-') - for dep in deps: print('\t| {:^50} | {:^50} |'.format(dep['name'], dep['folder'])) -else: - sys.exit("Error: Could not determine dependencies. At least the module itself should appear.") - -depNames = [dep['name'] for dep in deps] -depFolders = [dep['folder'] for dep in deps] -depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] -if cmdArgs['skipfolders']: - cmdArgs['skipfolders'] = [f.strip('/') for f in cmdArgs['skipfolders']] - depFolders = [d for d in depFolders if d not in cmdArgs['skipfolders']] - depNames = [getModuleInfo(d, 'Module') for d in depFolders] - depFolderPaths = [d for d in depFolderPaths if os.path.basename(d.strip('/')) not in cmdArgs['skipfolders']] - -################################# -# determine specific commits of all modules -print("\n-- Determining the module versions") -try: versions = getUsedVersions(depFolderPaths, cmdArgs['ignoreuntracked']) -except Exception as err: - print('\nCaught exception: ' + str(err)) - if 'untracked' in str(err): - print('If you are certain that the untracked files are not needed for the ' \ - 'installation of the modules, run this script with the -i flag.') - sys.exit(1) - -if len(versions) != len(depNames): - sys.exit("Error': Could not determine versions of all modules") - -print("-> Detected the following versions") -printVersionTable(versions) - -################################# -# create patches if necessary -print("\n-- Creating patches for unpublished commits and uncommitted changes") -patches = getPatches(depFolderPaths, cmdArgs['ignoreuntracked']) - -# create patch files -if len(patches) > 0: - patchesPath = os.path.join(modPath, 'patches') - if not os.path.exists(patchesPath): - os.mkdir(patchesPath) - print("-> Created a folder 'patches' in your module. You should commit this to your\n"\ - " repository in order for the installation script to work on other machines.") - else: - print("-> Adding patches to the 'patches' folder in your module. Make sure to commit\n"\ - " the newly added patches in order for the install script to work on other machines.") - - # get a non-used patch file name (makes sure not to overwrite anything) - def getPatchFileName(depModPath, targetName): - i = 1 - fileName = os.path.join(patchesPath, targetName + '.patch') - while os.path.exists(fileName): - fileName = os.path.join(patchesPath, targetName + '_' + str(i) + '.patch') - i += 1 - - return fileName, os.path.relpath(fileName, depModPath) - - # function to write a new patch file (returns the path to the new file) - def writeDepModPatchFile(depModPath, depModName, type): - patchPath, patchRelPath = getPatchFileName(depModPath, depModName + '_' + type) - patches[depModPath][type + '_relpath'] = patchRelPath - open(patchPath, 'w').write(patches[depModPath][type]) - return patchPath - - print("-> Created patch files:") - for depModPath in patches.keys(): - depModName = getModuleInfo(depModPath, "Module") - if 'unpublished' in patches[depModPath]: - print(' '*3 + writeDepModPatchFile(depModPath, depModName, 'unpublished')) - if 'uncommitted' in patches[depModPath]: - print(' '*3 + writeDepModPatchFile(depModPath, depModName, 'uncommitted')) - -else: - print("-> No Patches required") - - -################################## -# write installation shell script - -# in the install script, we work with relative paths from the parent to module folder -def getModRelPath(path): return os.path.relpath(path, modParentPath) -versions = {getModRelPath(modPath): val for modPath, val in versions.items()} -patches = {getModRelPath(modPath): val for modPath, val in patches.items()} - -topFolderName = cmdArgs['topfoldername'] -optsRelPath = 'dumux/cmake.opts' -if cmdArgs['optsfile']: - optsPath = os.path.abspath( os.path.join(os.getcwd(), cmdArgs['optsfile']) ) - optsRelPath = os.path.relpath(optsPath, modParentPath) - -# TODO: add support for different install script languages (e.g. Python). -with open(instFileName, 'w') as installFile: - installFile.write('#!/bin/bash\n\n') - installFile.write('#'*80 + '\n') - installFile.write('# This script installs the module "' + modName + '" together with all dependencies.\n') - installFile.write('\n') - - exitFunc = 'exitWith' - installFile.write('# defines a function to exit with error message\n') - installFile.write(exitFunc + ' ()\n') - installFile.write('{\n') - installFile.write(' echo ' + r'"\n$1"' +'\n') - installFile.write(' exit 1\n') - installFile.write('}\n\n') - - # function to write a command with error check into the script file - def writeCommand(command, errorMessage, indentationLevel = 0): - installFile.write(' '*indentationLevel + 'if ! ' + command + ';') - installFile.write(' then ' + exitFunc + ' "' + errorMessage + '"; fi\n') - - # write section on creating the top folder - if topFolderName: - installFile.write('# Everything will be installed into a newly created sub-folder named "' + topFolderName + '".\n') - installFile.write('echo "Creating the folder ' + topFolderName + ' to install the modules in"\n') - writeCommand('mkdir -p ' + topFolderName , '--Error: could not create top folder ' + topFolderName) - writeCommand('cd ' + topFolderName, '--Error: could not enter top folder ' + topFolderName) - installFile.write('\n') - else: - installFile.write('# Everything will be installed inside the folder from which the script is executed.\n\n') - - # function to write installation procedure for a module - def writeCloneModule(depModName, depModFolder): - installFile.write('# ' + depModName + '\n') - installFile.write('# ' + versions[depModFolder]['branch'] + ' # ' - + versions[depModFolder]['revision'] + ' # ' - + versions[depModFolder]['date'] + ' # ' - + versions[depModFolder]['author'] + '\n') - - writeCommand('git clone ' + versions[depModFolder]['remote'], - '-- Error: failed to clone ' + depModName + '.') - writeCommand('cd ' + depModFolder, - '-- Error: could not enter folder ' + depModFolder + '.') - writeCommand('git checkout ' + versions[depModFolder]['branch'], - '-- Error: failed to check out branch ' + versions[depModFolder]['branch'] + ' in module ' + depModName + '.') - writeCommand('git reset --hard ' + versions[depModFolder]['revision'], - '-- Error: failed to check out commit ' + versions[depModFolder]['revision'] + ' in module ' + depModName + '.') - - # write section on application of patches - def writeApplyPatch(patchRelPath): - installFile.write('if [ -f ' + patchRelPath + ' ]; then\n') - writeCommand('git apply ' + patchRelPath,'--Error: failed to apply patch ' + patchRelPath + ' in module ' + depModName + '.', 4) - installFile.write('else\n') - installFile.write(' '*4 + exitFunc + ' "--Error: patch ' + patchRelPath + ' was not found."\n') - installFile.write('fi\n') - - if depModFolder in patches and 'unpublished_relpath' in patches[depModFolder]: - writeApplyPatch(patches[depModFolder]['unpublished_relpath']) - if depModFolder in patches and 'uncommitted_relpath' in patches[depModFolder]: - writeApplyPatch(patches[depModFolder]['uncommitted_relpath']) - - installFile.write('echo "-- Successfully set up the module ' + depModName + r'\n"' + '\n') - installFile.write('cd ..\n\n') - - # write the module clone first in order for the patches to be present - if modName in depNames: - writeCloneModule(modName, modFolder) - for depModName, depModFolder in zip(depNames, depFolders): - if depModName != modName: - writeCloneModule(depModName, depModFolder) - - # write configure command - installFile.write('echo "-- All modules haven been cloned successfully. Configuring project..."\n') - writeCommand('./dune-common/bin/dunecontrol --opts=' + optsRelPath + ' all', '--Error: could not configure project') - - # build the tests of the module - installFile.write('\n') - installFile.write('echo "-- Configuring successful. Compiling applications..."\n') - writeCommand('cd ' + modFolder + "/build-cmake", '--Error: could not enter build directory at ' + modFolder + '/build-cmake') - writeCommand('make build_tests', '--Error: applications could not be compiled. Please try to compile them manually.') - -print("\n-- Successfully created install script file " + instFileName) - -if len(patches) > 0: - print("-> It is recommended that you now commit and publish the 'patches' folder and this install script in your module such that others can use it.") - print(" IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.") - print(" That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.") - print(" If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") - -if modName in depNames: # print gudience to installation if the module is not skipped - print("\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n") - print(" ## Installation\n") - print(" The easiest way of installation is to use the script `" + instFileName + "` provided in this repository.") - print(" Using `wget`, you can simply install all dependent modules by typing:\n") - print(" ```sh") - print(" wget " + versions[modFolder]['remote'] + "/" + instFileName) - print(" chmod u+x " + instFileName) - print(" ./" + instFileName) - print(" ```\n") - - if topFolderName: print(" This will create a sub-folder `" + topFolderName + "`, clone all modules into it, configure the entire project and build the applications contained in this module.") - else: print(" This will clone all modules into the folder from which the script is called, configure the entire project and build the applications contained in this module.") -- GitLab From 6211a8dcd821e3fb84d45b275a8e1eae5ef3760b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= Date: Mon, 31 May 2021 19:37:33 +0200 Subject: [PATCH 04/47] [bin][util] remove getusedversions The functionality (or similar) is now in bin/extractmodule/util.py --- bin/util/getusedversions.py | 136 ------------------------------------ 1 file changed, 136 deletions(-) delete mode 100755 bin/util/getusedversions.py diff --git a/bin/util/getusedversions.py b/bin/util/getusedversions.py deleted file mode 100755 index a9c933755b..0000000000 --- a/bin/util/getusedversions.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 - -import os -import argparse -from common import runCommand -from common import callFromPath - -# print warning message for scanned folders that are not git repositories -def printNoGitRepoWarning(folderPath): - print("Folder " + folderPath + " does not seem to be the top level" \ - "of a git repository and will be skipped. Make sure not to call " \ - "this script from a sub-directory of a git repository.") - -# raise error due to untracked files present in the given module folder -def raiseUntrackedFilesError(folderPath): - raise RuntimeError('Found untracked files in module folder: "' + folderPath + '". ' \ - 'Please commit, stash, or remove them.') - -# returns true if the given folder is a git repository -def isGitRepository(modFolderPath): - return os.path.exists(os.path.join(modFolderPath, '.git')) - - -# returns true if a module contains untracked files -def hasUntrackedFiles(modFolderPath): - run = callFromPath(modFolderPath)(runCommand) - return run('git ls-files --others --exclude-standard') != '' - - -# get the most recent commit that also exists on the remote master/release branch -# maybe used to find a commit we can use as basis for a pub module -def mostRecentCommonCommitWithRemote(modFolderPath, match=['/master', '/release']): - run = callFromPath(modFolderPath)(runCommand) - revList = run('git rev-list HEAD').split('\n') - for rev in revList: - remoteBranches = run('git branch -r --contains {}'.format(rev)) - for m in match: - if m in remoteBranches: - return rev - - raise RuntimeError('Could not suitable find ancestor commit' - ' that is synced with master on remote') - - -# function to extract git version information for modules -# returns a dictionary containing module information for each given module folder -def getUsedVersions(modFolderPaths, ignoreUntracked=False): - - result = {} - for modFolderPath in modFolderPaths: - - # make sure this is the top level of a git repository - if not isGitRepository(modFolderPath): - printNoGitRepoWarning(modFolderPath) - else: - if not ignoreUntracked and hasUntrackedFiles(modFolderPath): - raiseUntrackedFilesError(modFolderPath) - - run = callFromPath(modFolderPath)(runCommand) - result[modFolderPath] = {} - result[modFolderPath]['remote'] = run('git ls-remote --get-url').strip('\n') - # update remote to make sure we find all upstream commits - run('git fetch {}'.format(result[modFolderPath]['remote'])) - rev = mostRecentCommonCommitWithRemote(modFolderPaths) - result[modFolderPath]['revision'] = rev - result[modFolderPath]['date'] = run('git log -n 1 --format=%ai {}'.format(rev)).strip('\n') - result[modFolderPath]['author'] = run('git log -n 1 --format=%an {}'.format(rev)).strip('\n') - # this may return HEAD if we are on some detached HEAD tree - result[modFolderPath]['branch'] = run('git rev-parse --abbrev-ref HEAD').strip('\n') - - return result - -# create patches for unpublished commits and uncommitted changes in modules -def getPatches(modFolderPaths, ignoreUntracked=False): - - result = {} - for modFolderPath in modFolderPaths: - - # make sure this is the top level of a git repository - if not isGitRepository(modFolderPath): - printNoGitRepoWarning(modFolderPath) - else: - if not ignoreUntracked and hasUntrackedFiles(modFolderPath): - raiseUntrackedFilesError(modFolderPath) - - run = callFromPath(modFolderPath)(runCommand) - unpubPatch = run('git format-patch --stdout @{upstream}') - unCommPatch = run('git diff') - if unpubPatch != '' or unCommPatch != '': result[modFolderPath] = {} - if unpubPatch != '': result[modFolderPath]['unpublished'] = unpubPatch - if unCommPatch != '': result[modFolderPath]['uncommitted'] = unCommPatch - - return result - -# prints the detected versions as table -def printVersionTable(versions): - print("\t| {:^50} | {:^50} | {:^50} | {:^30} |".format('module folder', 'branch', 'commit hash', 'commit date')) - print("\t" + 193*'-') - for folder, versionInfo in versions.items(): - print("\t| {:^50} | {:^50} | {:^50} | {:^30} |".format(folder, versionInfo['branch'], versionInfo['revision'], versionInfo['date'])) - - -# For standalone execution -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='This script extracts the used dune/dumux versions.') - parser.add_argument('-p', '--path', required=False, help='the path to the top folder containing your dune/dumux modules') - parser.add_argument('-i', '--ignoreuntracked', required=False, action='store_true', help='use this flag to ignore untracked files present in the modules') - parser.add_argument('-s', '--skipfolders', required=False, nargs='*', help='a list of module folders to be skipped') - cmdArgs = vars(parser.parse_args()) - - modulesPath = os.getcwd() if not cmdArgs['path'] else os.path.join(os.getcwd(), cmdArgs['path']) - print('\nDetermining the versions of all dune modules in the folder: ' + modulesPath) - - def getPath(modFolder): - return os.path.join(modulesPath, modFolder) - - modFolderPaths = [getPath(dir) for dir in os.listdir(modulesPath) if os.path.isdir(getPath(dir))] - if cmdArgs['skipfolders']: - cmdArgs['skipfolders'] = [f.strip('/') for f in cmdArgs['skipfolders']] - modFolderPaths = [d for d in modFolderPaths if os.path.basename(d.strip('/')) not in cmdArgs['skipfolders']] - versions = getUsedVersions(modFolderPaths, True) - - print("\nDetected the following versions:") - printVersionTable(versions) - - # maybe check untracked files - if not cmdArgs['ignoreuntracked']: - modsWithUntracked = [f for f in versions if hasUntrackedFiles(f)] - if modsWithUntracked: - print('\n') - print('#'*56) - print('WARNING: Found untracked files in the following modules:\n\n') - print('\n'.join(modsWithUntracked)) - print('\nPlease make sure that these are not required for your purposes.') - print('If not, you can run this script with the option -i/--ignoreuntracked to suppress this warning.') - print('#'*56) -- GitLab From 3325bbcb63f3ceba3d542abc18a77d2fbb2de94e Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Mon, 31 May 2021 18:26:39 +0000 Subject: [PATCH 05/47] [scripts][makeinstall] Extract main function --- bin/extractmodule/makeinstallscript.py | 564 +++++++++++++------------ 1 file changed, 289 insertions(+), 275 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 681f36e7fe..2ed5845081 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -23,301 +23,315 @@ if sys.version_info[0] < 3: sys.exit("\nError': Python3 required") -################### -# parse arguments -parser = argparse.ArgumentParser( - description='This script generates an install script for your dune module,' - 'taking into account non-published commits & local changes.\n' - 'This expects that your module is a git repository and that a ' - 'remote origin exists and has been set already.' -) -parser.add_argument('-p', '--path', - required=True, - help='The path to your dune module') -parser.add_argument('-f', '--filename', - required=False, - help='File in which to write the install script') -parser.add_argument('-i', '--ignoreuntracked', - required=False, action='store_true', - help='Use this to ignore untracked files present') -parser.add_argument('-t', '--topfoldername', - required=False, default='DUMUX', - help='Name of the folder that the install script creates ' - 'upon execution to install the modules in. If you ' - 'pass an empty string, no folder will be created ' - 'and installation happens in place.') -parser.add_argument('-o', '--optsfile', - required=False, - help='Provide custom opts file to be used for the call to ' - 'dunecontrol. Note that this file is required to be ' - 'contained and committed within the module or its ' - 'dependencies.') -cmdArgs = vars(parser.parse_args()) - - -cwd = os.getcwd() -modPath = os.path.abspath(os.path.join(cwd, cmdArgs['path'])) -modParentPath = os.path.abspath(os.path.join(modPath, '../')) -modFolder = os.path.relpath(modPath, modParentPath) +def makeInstallScript(path, + fileName=None, + ignoreUntracked=False, + topFolderName='DUMUX', + optsFile=None): + + cwd = os.getcwd() + modPath = os.path.abspath(os.path.join(cwd, path)) + modParentPath = os.path.abspath(os.path.join(modPath, '../')) + modFolder = os.path.relpath(modPath, modParentPath) + + try: + modName = getModuleInfo(modPath, 'Module') + except Exception: + sys.exit("\nError: Could not determine module name. Make sure the module\n" + " path is correct and that it contains 'dune.module'.") + + if not fileName: + instFileName = 'install_' + modName + '.sh' + else: + instFileName = fileName + print("\n-- Creating install script '{}' for module '{}' in folder '{}'" + .format(instFileName, modName, modPath)) + + + print("\n-- Determining the dependencies") + deps = getDependencies(modPath) + if len(deps) > 0: + print("-> Found the following dependencies") + print("\t| {:^50} | {:^50} |".format('module name', 'module folder')) + print("\t" + 107*'-') + for dep in deps: + print('\t| {:^50} | {:^50} |'.format(dep['name'], dep['folder'])) + else: + sys.exit("Error: Could not determine module dependencies.") + + depNames = [dep['name'] for dep in deps] + depFolders = [dep['folder'] for dep in deps] + depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] + + + print("\n-- Determining the module versions") + try: + versions = getPersistentVersions(depFolderPaths, ignoreUntracked) + except Exception as err: + print('\nCaught exception: ' + str(err)) + if 'untracked' in str(err): + print('If you are certain that the untracked files are not needed for ' + 'the module installation, run this script with the -i flag.') + sys.exit(1) + + if len(versions) != len(depNames): + sys.exit("Error': Could not determine versions of all modules") + + print("-> The following (remotely available) versions are used as basis") + print(" on top of which we will generate the required patches") + printVersionTable(versions) + + + print("\n-- Creating patches for unpublished commits and uncommitted changes") + patches = getPatches(versions) + + # create patch files + if len(patches) > 0: + print("-> Placing patches in the folder 'patches' in your module. You " + "should commit them to your repository in order for the install" + "script to work on other machines.") + patchesPath = os.path.join(modPath, 'patches') + os.makedirs(patchesPath, exist_ok=True) + + def getPatchFileNameAndRelPath(depModPath, targetName): + i = 1 + fileName = os.path.join(patchesPath, targetName + '.patch') + while os.path.exists(fileName): + fileName = os.path.join(patchesPath, + targetName + '_' + str(i) + '.patch') + i += 1 + + return fileName, os.path.relpath(fileName, depModPath) + + def writePatch(depModPath, depModName, type): + patchName = depModName + '_' + type + pPath, pRelPath = getPatchFileNameAndRelPath(depModPath, patchName) + patches[depModPath][type + '_relpath'] = pRelPath + open(pPath, 'w').write(patches[depModPath][type]) + return pPath + + print("-> Created patch files:") + for depModPath in patches.keys(): + depModName = getModuleInfo(depModPath, "Module") + if 'unpublished' in patches[depModPath]: + print(' '*3 + writePatch(depModPath, depModName, 'unpublished')) + if 'uncommitted' in patches[depModPath]: + print(' '*3 + writePatch(depModPath, depModName, 'uncommitted')) -try: - modName = getModuleInfo(modPath, 'Module') -except Exception: - sys.exit("\nError: Could not determine module name. Make sure the module\n" - " path is correct and that it contains 'dune.module'.") - -if not cmdArgs['filename']: - instFileName = 'install_' + modName + '.sh' -else: - instFileName = cmdArgs['filename'] -print("\n-- Creating install script '{}' for module '{}' in folder '{}'" - .format(instFileName, modName, modPath)) - - -print("\n-- Determining the dependencies") -deps = getDependencies(modPath) -if len(deps) > 0: - print("-> Found the following dependencies") - print("\t| {:^50} | {:^50} |".format('module name', 'module folder')) - print("\t" + 107*'-') - for dep in deps: - print('\t| {:^50} | {:^50} |'.format(dep['name'], dep['folder'])) -else: - sys.exit("Error: Could not determine module dependencies.") - -depNames = [dep['name'] for dep in deps] -depFolders = [dep['folder'] for dep in deps] -depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] - - -print("\n-- Determining the module versions") -try: - ignoreUntracked = cmdArgs['ignoreuntracked'] - versions = getPersistentVersions(depFolderPaths, ignoreUntracked) -except Exception as err: - print('\nCaught exception: ' + str(err)) - if 'untracked' in str(err): - print('If you are certain that the untracked files are not needed for ' - 'the module installation, run this script with the -i flag.') - sys.exit(1) - -if len(versions) != len(depNames): - sys.exit("Error': Could not determine versions of all modules") - -print("-> The following (remotely available) versions are used as basis") -print(" on top of which we will generate the required patches") -printVersionTable(versions) - - -print("\n-- Creating patches for unpublished commits and uncommitted changes") -patches = getPatches(versions) - -# create patch files -if len(patches) > 0: - print("-> Placing patches in the folder 'patches' in your module. You " - "should commit them to your repository in order for the install" - "script to work on other machines.") - patchesPath = os.path.join(modPath, 'patches') - os.makedirs(patchesPath, exist_ok=True) - - def getPatchFileNameAndRelPath(depModPath, targetName): - i = 1 - fileName = os.path.join(patchesPath, targetName + '.patch') - while os.path.exists(fileName): - fileName = os.path.join(patchesPath, - targetName + '_' + str(i) + '.patch') - i += 1 - - return fileName, os.path.relpath(fileName, depModPath) - - def writePatch(depModPath, depModName, type): - patchName = depModName + '_' + type - pPath, pRelPath = getPatchFileNameAndRelPath(depModPath, patchName) - patches[depModPath][type + '_relpath'] = pRelPath - open(pPath, 'w').write(patches[depModPath][type]) - return pPath - - print("-> Created patch files:") - for depModPath in patches.keys(): - depModName = getModuleInfo(depModPath, "Module") - if 'unpublished' in patches[depModPath]: - print(' '*3 + writePatch(depModPath, depModName, 'unpublished')) - if 'uncommitted' in patches[depModPath]: - print(' '*3 + writePatch(depModPath, depModName, 'uncommitted')) - -else: - print("-> No Patches required") - - -# write installation shell script (switch to relative paths) -versions = {os.path.relpath(p, modParentPath): v for p, v in versions.items()} -patches = {os.path.relpath(p, modParentPath): v for p, v in patches.items()} - -topFolderName = cmdArgs['topfoldername'] -optsRelPath = 'dumux/cmake.opts' -if cmdArgs['optsfile']: - optsPath = os.path.abspath(os.path.join(os.getcwd(), cmdArgs['optsfile'])) - optsRelPath = os.path.relpath(optsPath, modParentPath) - -# TODO: add support for different install script languages (e.g. Python). -with open(instFileName, 'w') as installFile: - installFile.write("#!/bin/bash\n\n") - installFile.write("#"*80 + "\n") - installFile.write("# This script installs the module '{}'" - " together with all dependencies.\n" - "\n".format(modName)) - - exitFunc = 'exitWith' - installFile.write( - '# defines a function to exit with error message\n' - '{} ()\n'.format(exitFunc) - + - '{\n' - ' echo "\\n$1"\n' - ' exit 1\n' - '}\n\n' - ) + else: + print("-> No Patches required") - unitIndentation = ' '*4 - def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): - indent = unitIndentation*indentationLevel - nextIndent = indent + unitIndentation - installFile.write( - f"{indent}if ! {command}; then\n" - f"{nextIndent}echo \"{errorMessage}\"\n" - "fi\n" - ) + # write installation shell script (switch to relative paths) + versions = {os.path.relpath(p, modParentPath): v for p, v in versions.items()} + patches = {os.path.relpath(p, modParentPath): v for p, v in patches.items()} - if topFolderName: + optsRelPath = 'dumux/cmake.opts' + if optsFile: + optsPath = os.path.abspath(os.path.join(os.getcwd(), optsFile)) + optsRelPath = os.path.relpath(optsPath, modParentPath) + + # TODO: add support for different install script languages (e.g. Python). + with open(instFileName, 'w') as installFile: + installFile.write("#!/bin/bash\n\n") + installFile.write("#"*80 + "\n") + installFile.write("# This script installs the module '{}'" + " together with all dependencies.\n" + "\n".format(modName)) + + exitFunc = 'exitWith' installFile.write( - '# Everything will be installed into a new folder "{}".\n' - .format(topFolderName) + '# defines a function to exit with error message\n' + '{} ()\n'.format(exitFunc) + + + '{\n' + ' echo "\\n$1"\n' + ' exit 1\n' + '}\n\n' ) - installFile.write( - 'echo "Creating the folder {} to install the modules in"\n' - .format(topFolderName) - ) + unitIndentation = ' '*4 - writeCommandWithErrorCheck( - 'mkdir -p {}'.format(topFolderName), - '--Error: could not create top folder {}'.format(topFolderName) - ) + def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): + indent = unitIndentation*indentationLevel + nextIndent = indent + unitIndentation + installFile.write( + f"{indent}if ! {command}; then\n" + f"{nextIndent}echo \"{errorMessage}\"\n" + "fi\n" + ) - writeCommandWithErrorCheck( - 'cd {}'.format(topFolderName), - '--Error: could not enter top folder {}'.format(topFolderName)) + if topFolderName: + installFile.write( + '# Everything will be installed into a new folder "{}".\n' + .format(topFolderName) + ) + installFile.write( + 'echo "Creating the folder {} to install the modules in"\n' + .format(topFolderName) - installFile.write('\n') - else: - installFile.write( - '# Everything will be installed inside the folder from which the' - ' script is executed.\n\n' - ) + ) - def writeCloneModule(depModName, depModFolder): - installFile.write('# ' + depModName + '\n') - installFile.write('# ' + versions[depModFolder]['branch'] + ' # ' - + versions[depModFolder]['revision'] + ' # ' - + versions[depModFolder]['date'] + ' # ' - + versions[depModFolder]['author'] + '\n') + writeCommandWithErrorCheck( + 'mkdir -p {}'.format(topFolderName), + '--Error: could not create top folder {}'.format(topFolderName) + ) + writeCommandWithErrorCheck( + 'cd {}'.format(topFolderName), + '--Error: could not enter top folder {}'.format(topFolderName)) + + installFile.write('\n') + else: + installFile.write( + '# Everything will be installed inside the folder from which the' + ' script is executed.\n\n' + ) + + def writeCloneModule(depModName, depModFolder): + installFile.write('# ' + depModName + '\n') + installFile.write('# ' + versions[depModFolder]['branch'] + ' # ' + + versions[depModFolder]['revision'] + ' # ' + + versions[depModFolder]['date'] + ' # ' + + versions[depModFolder]['author'] + '\n') + + writeCommandWithErrorCheck( + 'git clone {}'.format(versions[depModFolder]['remote']), + '-- Error: failed to clone {}.'.format(depModName) + ) + writeCommandWithErrorCheck( + 'cd {}'.format(depModFolder), + '-- Error: could not enter folder {}.'.format(depModFolder) + ) + writeCommandWithErrorCheck( + 'git checkout {}'.format(versions[depModFolder]['branch']), + '-- Error: failed to check out branch {} in module {}.' + .format(versions[depModFolder]['branch'], depModName) + ) + writeCommandWithErrorCheck( + 'git reset --hard {}'.format(versions[depModFolder]['revision']), + '-- Error: failed to check out commit {} in module {}.' + .format(versions[depModFolder]['revision'], depModName) + ) + + # write section on application of patches + def writeApplyPatch(patchRelPath): + patchError = '--Error: patch {} was not found'.format(patchRelPath) + installFile.write('if [ -f {} ]; then\n'.format(patchRelPath)) + writeCommandWithErrorCheck( + 'git apply {}'.format(patchRelPath), + '--Error: failed to apply patch {} in module {}' + .format(patchRelPath, depModName), + indentationLevel=1 + ) + installFile.write("else\n" + f"{unitIndentation}{exitFunc} " + f"\"{patchError}\".\n") + installFile.write('fi\n') + + if depModFolder in patches: + if'unpublished_relpath' in patches[depModFolder]: + writeApplyPatch(patches[depModFolder]['unpublished_relpath']) + if 'uncommitted_relpath' in patches[depModFolder]: + writeApplyPatch(patches[depModFolder]['uncommitted_relpath']) + + installFile.write('echo "-- Successfully set up the module {} \\n"\n' + .format(depModName) + '\n') + installFile.write('cd ..\n\n') + + # write the module clone first in order for the patches to be present + if modName in depNames: + writeCloneModule(modName, modFolder) + for depModName, depModFolder in zip(depNames, depFolders): + if depModName != modName: + writeCloneModule(depModName, depModFolder) + + # write configure command + installFile.write('echo "-- All modules haven been cloned successfully. ' + 'Configuring project..."\n') writeCommandWithErrorCheck( - 'git clone {}'.format(versions[depModFolder]['remote']), - '-- Error: failed to clone {}.'.format(depModName) + './dune-common/bin/dunecontrol --opts={} all'.format(optsRelPath), + '--Error: could not configure project' ) - writeCommandWithErrorCheck( - 'cd {}'.format(depModFolder), - '-- Error: could not enter folder {}.'.format(depModFolder) + + # build the tests of the module + installFile.write( + '\n' + 'echo "-- Configuring successful. Compiling applications..."\n' ) writeCommandWithErrorCheck( - 'git checkout {}'.format(versions[depModFolder]['branch']), - '-- Error: failed to check out branch {} in module {}.' - .format(versions[depModFolder]['branch'], depModName) + 'cd {}/build-cmake'.format(modFolder), + '--Error: could not enter build directory at {}/build-cmake' + .format(modFolder) ) writeCommandWithErrorCheck( - 'git reset --hard {}'.format(versions[depModFolder]['revision']), - '-- Error: failed to check out commit {} in module {}.' - .format(versions[depModFolder]['revision'], depModName) + 'make build_tests', + '--Error: applications could not be compiled. ' + 'Please try to compile them manually.' ) - # write section on application of patches - def writeApplyPatch(patchRelPath): - patchError = '--Error: patch {} was not found'.format(patchRelPath) - installFile.write('if [ -f {} ]; then\n'.format(patchRelPath)) - writeCommandWithErrorCheck( - 'git apply {}'.format(patchRelPath), - '--Error: failed to apply patch {} in module {}' - .format(patchRelPath, depModName), - indentationLevel=1 - ) - installFile.write("else\n" - f"{unitIndentation}{exitFunc} " - f"\"{patchError}\".\n") - installFile.write('fi\n') - - if depModFolder in patches: - if'unpublished_relpath' in patches[depModFolder]: - writeApplyPatch(patches[depModFolder]['unpublished_relpath']) - if 'uncommitted_relpath' in patches[depModFolder]: - writeApplyPatch(patches[depModFolder]['uncommitted_relpath']) - - installFile.write('echo "-- Successfully set up the module {} \\n"\n' - .format(depModName) + '\n') - installFile.write('cd ..\n\n') - - # write the module clone first in order for the patches to be present - if modName in depNames: - writeCloneModule(modName, modFolder) - for depModName, depModFolder in zip(depNames, depFolders): - if depModName != modName: - writeCloneModule(depModName, depModFolder) - - # write configure command - installFile.write('echo "-- All modules haven been cloned successfully. ' - 'Configuring project..."\n') - writeCommandWithErrorCheck( - './dune-common/bin/dunecontrol --opts={} all'.format(optsRelPath), - '--Error: could not configure project' - ) + print("\n-- Successfully created install script file " + instFileName) + + if len(patches) > 0: + print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" + " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" + " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" + " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") + + print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" + f" ## Installation\n" + f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" + f" Using `wget`, you can simply install all dependent modules by typing:\n" + f"\n" + f" ```sh\n" + f" wget {versions[modFolder]['remote']}/{instFileName}\n" + f" chmod u+x {instFileName}\n" + f" ./{instFileName}\n" + f" ```\n") - # build the tests of the module - installFile.write( - '\n' - 'echo "-- Configuring successful. Compiling applications..."\n' - ) - writeCommandWithErrorCheck( - 'cd {}/build-cmake'.format(modFolder), - '--Error: could not enter build directory at {}/build-cmake' - .format(modFolder) + if topFolderName: + print(f" This will create a sub-folder `{topFolderName}`, clone all modules into it, configure the entire project and build the applications contained in this module.") + else: + print(" This will clone all modules into the folder from which the script is called, configure the entire project and build the applications contained in this module.") + + +if __name__ == '__main__': + + ################### + # parse arguments + parser = argparse.ArgumentParser( + description='This script generates an install script for your dune module,' + 'taking into account non-published commits & local changes.\n' + 'This expects that your module is a git repository and that a ' + 'remote origin exists and has been set already.' ) - writeCommandWithErrorCheck( - 'make build_tests', - '--Error: applications could not be compiled. ' - 'Please try to compile them manually.' + parser.add_argument('-p', '--path', + required=True, + help='The path to your dune module') + parser.add_argument('-f', '--filename', + required=False, + help='File in which to write the install script') + parser.add_argument('-i', '--ignoreuntracked', + required=False, action='store_true', + help='Use this to ignore untracked files present') + parser.add_argument('-t', '--topfoldername', + required=False, default='DUMUX', + help='Name of the folder that the install script creates ' + 'upon execution to install the modules in. If you ' + 'pass an empty string, no folder will be created ' + 'and installation happens in place.') + parser.add_argument('-o', '--optsfile', + required=False, + help='Provide custom opts file to be used for the call to ' + 'dunecontrol. Note that this file is required to be ' + 'contained and committed within the module or its ' + 'dependencies.') + cmdArgs = vars(parser.parse_args()) + + makeInstallScript( + path=cmdArgs['path'], + fileName=cmdArgs.get('filename', None), + ignoreUntracked=cmdArgs.get('ignoreuntracked', False), + topFolderName=cmdArgs.get('topfoldername', None), + optsFile=cmdArgs.get('optsFile', None) ) - -print("\n-- Successfully created install script file " + instFileName) - -if len(patches) > 0: - print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" - " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" - " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" - " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") - -print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" - f" ## Installation\n" - f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" - f" Using `wget`, you can simply install all dependent modules by typing:\n" - f"\n" - f" ```sh\n" - f" wget {versions[modFolder]['remote']}/{instFileName}\n" - f" chmod u+x {instFileName}\n" - f" ./{instFileName}\n" - f" ```\n") - -if topFolderName: - print(f" This will create a sub-folder `{topFolderName}`, clone all modules into it, configure the entire project and build the applications contained in this module.") -else: - print(" This will clone all modules into the folder from which the script is called, configure the entire project and build the applications contained in this module.") -- GitLab From 3a4664cdda51be1e21321ae59eac66190d0166bc Mon Sep 17 00:00:00 2001 From: hanchuan <> Date: Thu, 3 Jun 2021 18:53:11 +0200 Subject: [PATCH 06/47] Improved extract module part script --- .../extract_as_new_module.py} | 291 ++++++++++++------ .../extractmodulepart.sh | 0 2 files changed, 200 insertions(+), 91 deletions(-) rename bin/{util/extractmodulepart.py => extractmodule/extract_as_new_module.py} (55%) rename bin/{util => extractmodule}/extractmodulepart.sh (100%) diff --git a/bin/util/extractmodulepart.py b/bin/extractmodule/extract_as_new_module.py similarity index 55% rename from bin/util/extractmodulepart.py rename to bin/extractmodule/extract_as_new_module.py index fbd7cfc692..99e4d3d9a6 100755 --- a/bin/util/extractmodulepart.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 -import sys, glob, os, subprocess +""" +This script extracts some specified applications into a separate Dune module. +For example make a dumux-pub repository accompanying a scientific paper. +""" +import sys +import os +import subprocess import os.path import argparse import shutil @@ -7,15 +13,18 @@ from distutils.dir_util import copy_tree import re import threading import fnmatch -from getmoduleinfo import * -from getusedversions import * - - -""" -This is a python script for extracting module -""" - -# check if help is needed +from util import getPersistentVersions +from util import printVersionTable +from makeinstallscript import makeInstallScript +try: + path = os.path.split(os.path.abspath(__file__))[0] + sys.path.append(os.path.join(path, '../bin/util')) + from getmoduleinfo import getDependencies + from common import callFromPath, runCommand +except Exception: + sys.exit('Could not import common modul or getModuleInfo') + +# set script parameters epilog = ''' ----------------------------------------------------------- The script has to be called one level above module_dir. @@ -23,22 +32,27 @@ At least one of the subfolders (FOLDER_1 [FOLDER_2 ...]) has to contain a source file *.cc of an executable for which you would like to timber a table in dumux-pub.) ''' -parser = argparse.ArgumentParser(prog='extractmodulepart', - usage= "./extractmodulepart module_dir SUBFOLDER_1 [SUBFOLDER_2 ...]", - description='This script extracts a subfolder of a DUNE module', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=epilog) -parser.add_argument('module_dir', help='Dune module from which the subfolder is extracted') -parser.add_argument('subfolder', nargs='+', help = 'subfolder(s) of module_dir which you want to extract') +parser = argparse.ArgumentParser( + prog='extractmodulepart', + usage="./extractmodulepart module_dir SUBFOLDER_1 [SUBFOLDER_2 ...]", + description='This script extracts a subfolder of a DUNE module', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=epilog) +parser.add_argument('module_dir', + help='Dune module from which the subfolder is extracted') +parser.add_argument('subfolder', + nargs='+', + help='subfolder(s) of module_dir that you want to extract') args = vars(parser.parse_args()) -# function to search the header file efficiently with parallel programming + +# function to search matching header files def search_headers(c_file): with open(c_file, 'r') as f: content = f.read() - header_in_bracket = re.findall(r'#include\s+<(.+?)>',content) - header_in_quotation = re.findall(r'#include\s+"(.+?)"',content) + header_in_bracket = re.findall(r'#include\s+<(.+?)>', content) + header_in_quotation = re.findall(r'#include\s+"(.+?)"', content) # search for includes relative to the module path for header in header_in_bracket + header_in_quotation: @@ -47,7 +61,8 @@ def search_headers(c_file): continue if header_with_path not in all_headers: all_headers.append(header_with_path) - thread_ = threading.Thread(target = search_headers, args = (header_with_path, )) + thread_ = threading.Thread(target=search_headers, + args=(header_with_path, )) thread_.start() # search for includes relative to the path of the including file @@ -61,10 +76,12 @@ def search_headers(c_file): continue if header_with_path not in all_headers: all_headers.append(header_with_path) - thread_ = threading.Thread(target = search_headers, args = (header_with_path, )) + thread_ = threading.Thread(target=search_headers, + args=(header_with_path, )) thread_.start() -# functions to find the file with specific pattern or name in a direcotry + +# functions to find the file with specific name in a direcotry def find_files_with_name(name, path): result = [] for root, dirs, files in os.walk(path): @@ -72,6 +89,8 @@ def find_files_with_name(name, path): result.append(os.path.join(root, name)) return result + +# functions to find the file with specific pattern in a direcotry def find_files_with_pattern(pattern, path): result = [] for root, dirs, files in os.walk(path): @@ -80,6 +99,7 @@ def find_files_with_pattern(pattern, path): result.append(os.path.join(root, name)) return result + # functions to configure CMakeLists.txt files def generate_new_content(lines, pattern, replace_str): index = lines.find(pattern) @@ -111,21 +131,25 @@ def generate_new_content(lines, pattern, replace_str): content = lines + "\n" + replace_str return content + def generate_subdirectory_content(dirs): content = "" for name in dirs: content += "add_subdirectory(" + name + ")" + "\n" return content + def generate_install_content(header_files, destination): if len(header_files) == 0: return "" content = "install(FILES" + "\n" for header_file in header_files: content += " " + header_file + "\n" - content += "DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}" + destination + ")" + "\n" + content += ("DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}" + + destination + ")" + "\n") return content + def generate_content(cmake_list_txt_file, dirs, header_files, destination): subdirectory_content = generate_subdirectory_content(dirs) install_content = generate_install_content(header_files, destination) @@ -134,7 +158,8 @@ def generate_content(cmake_list_txt_file, dirs, header_files, destination): with open(cmake_list_txt_file, "r", encoding="utf-8") as f: content = "".join(f.readlines()).strip() f.close() - content = generate_new_content(content, "add_subdirectory(", subdirectory_content) + content = generate_new_content(content, "add_subdirectory(", + subdirectory_content) return generate_new_content(content, "install(FILE", install_content) else: if subdirectory_content == "": @@ -142,13 +167,16 @@ def generate_content(cmake_list_txt_file, dirs, header_files, destination): else: return subdirectory_content + "\n" + install_content -def generate_cmake_lists_txt(cmake_list_txt_file, dirs, header_files, destination): - content = generate_content(cmake_list_txt_file, dirs, header_files, destination) + +def generate_cmake_lists_txt(cmake_list_txt_file, dirs, + header_files, destination): + content = generate_content(cmake_list_txt_file, dirs, + header_files, destination) with open(cmake_list_txt_file, "w", encoding="utf-8") as f: if content != "": f.write(content) - f.close() + def check_dir(root_dir): if not os.path.exists(root_dir): @@ -159,6 +187,7 @@ def check_dir(root_dir): return False return True + def check_str(root_dir): if root_dir is None or root_dir.strip() == "": return None @@ -167,6 +196,7 @@ def check_str(root_dir): root_dir = root_dir[:-1] return root_dir + def generate_cmake_lists_txt_file(root_dir): root_dir = check_str(root_dir) if root_dir is None: @@ -182,8 +212,11 @@ def generate_cmake_lists_txt_file(root_dir): for suffix in [".h", ".hh"]: if name.endswith(suffix): header_files.append(name) - generate_cmake_lists_txt(cmake_list_txt_file, dirs, header_files, destination) + generate_cmake_lists_txt(cmake_list_txt_file, dirs, + header_files, destination) + +# function asking user to answer yes or no question def query_yes_no(question, default="yes"): valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} if default is None: @@ -202,7 +235,29 @@ def query_yes_no(question, default="yes"): elif choice in valid: return valid[choice] else: - sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + sys.stdout.write("Please respond with 'yes' or 'no' " + "(or 'y' or 'n').\n") + + +# funtion to get correct git remote url and make sure it is empty +def get_remote_url(repo_path): + run_from_mod = callFromPath(repo_path)(runCommand) + while True: + if query_yes_no("Do you own a subfolder in dumux-pub?"): + nameyearx = input("Type below the information of" + " AuthorLastNameYearx (e.g. luggi2020b)\n") + remoteurl = 'https://git.iws.uni-stuttgart.de'\ + '/dumux-pub/{}.git'.format(nameyearx) + else: + remoteurl = input("Provide URL of your remote repository:\n") + check_remote_repo = run_from_mod('git ls-remote {}'.format(remoteurl)) + if (check_remote_repo is None): + sys.stdout.write("\nERROR: Please re-enter correct information.\n") + elif (check_remote_repo == ''): + return remoteurl + else: + sys.stdout.write("ERROR: The remote reposity is not empty!.\n") + if __name__ == "__main__": @@ -211,21 +266,22 @@ if __name__ == "__main__": # if module_dir contains a slash as last character, delete it if module_dir.endswith('/'): - module_dir = module_dir[:-1] + module_dir = module_dir[:-1] # check if we are above module_dir if not (os.path.isdir(module_dir)): - print("ERROR: you need to run the script one level above the folder "+module_dir+".") + print("ERROR: you need to run the script one level above the folder " + + module_dir + ".") print("Run \""+os.path.basename(__file__)+" --help\" for details.") exit(1) # determine all source files in the paths passed as arguments script_path = os.getcwd() os.chdir(module_dir) - module_full_path=os.getcwd() - all_sources=[] + module_full_path = os.getcwd() + all_sources = [] all_sources_with_path = [] - all_directories=[] + all_directories = [] for dir_path in subfolders: if dir_path.startswith(module_dir): # the "+1" is required to also remove the "/" @@ -244,54 +300,53 @@ if __name__ == "__main__": sourceswithpath = os.path.join(module_full_path, sources) all_sources_with_path.append(sourceswithpath) os.chdir(module_full_path) - os.chdir("..") # back to the script folder + os.chdir("..") # back to the script folder # check if sources have been obtained if (all_sources == []): - print("ERROR: no source files *.cc found in the directories "+ " ".join([str(x) for x in subfolders]) +".") - print("Be sure to provide a list of paths as arguments to this script.") + print("ERROR: no source files *.cc found in the directories " + + " ".join([str(x) for x in subfolders]) + ".") + print("Be sure to provide a list of paths as arguments.") print("Run \""+os.path.basename(__file__)+" --help\" for details.") exit(1) # try to find the duneproject script - dune_project = shutil.which('duneproject', path = "dune-common/bin") - if (dune_project == None): + dune_project = shutil.which('duneproject', path="dune-common/bin") + if (dune_project is None): print("ERROR: Could not find duneproject.") - print("Be sure to either have duneproject in your search path") - print("or to run this script from a directory that contains duneproject.") + print("Be sure to have duneproject in dune-common/bin") exit(1) else: print(dune_project) # give explanations print("\n""This script will\n" - "- extract the following sub-folders of "+ module_dir +":\n") + "- extract the following sub-folders of " + module_dir + ":\n") for dir_path in all_directories: - print(" "+ dir_path + ",") - print("\n"" and all headers in "+ module_dir + " that are required to build the\n" + print(" " + dir_path + ",") + print("\n"" and all headers in " + module_dir + + " that are required to build the\n" " executables from the sources\n") for source in all_sources: - print(" "+ source +",") + print(" " + source + ",") print(""" -- copy the extracted files into a freshly created DUNE module, retaining the directory structure, - -- update/create all required files like CMakeLists.txt, - store the versions of all used Dune module - -- and extract their modifications as patches. - -Thus, you receive a fully-working DUNE module containing the subset of {0} that is required to run your application. -duneproject will be run now. The new module should NOT depend on the module in {0}.\n\n""".format(module_dir)) +This script copies extracted files into a freshly created DUNE module +retaining the directory structure +and updates/creates required files like CMakeLists.txt. +duneproject will run now. +The new module should NOT depend on the module in {0}.\n\n""" + .format(module_dir)) input("Read the above and press [Enter] to proceed...") # run duneproject old_ls = os.listdir() subprocess.call([dune_project]) - new_ls = os.listdir() + new_ls = os.listdir() # determine the new module/directory name module_name = (set(new_ls) - set(old_ls)).pop() - if (module_name == ""): + if (module_name == ""): print("ERROR: could not find new module. Aborting.") exit(1) else: @@ -299,30 +354,36 @@ duneproject will be run now. The new module should NOT depend on the module in { print(os.path.basename(__file__) + ": Found new module " + module_name) print("Determining required headers...") os.chdir(module_name) - module_path=os.getcwd() + module_path = os.getcwd() - # extract all headers, provide some output and copy everything to the new module + # extract headers, provide some output and copy everything to new module all_headers = [] print("The following header files are extracted: ") for source in all_sources_with_path: search_headers(source) for header in all_headers: print(header) - dir_path = os.path.dirname(os.path.realpath(header)).replace(module_dir,module_name,1) + dir_path = os.path.dirname(os.path.realpath(header))\ + .replace(module_dir, module_name, 1) os.makedirs(dir_path, exist_ok=True) - shutil.copy(header,dir_path) + shutil.copy(header, dir_path) source_dir = os.path.dirname(source) - source_path = source_dir.replace(module_dir,module_name,1) + source_path = source_dir.replace(module_dir, module_name, 1) copy_tree(source_dir, source_path) + # copy .gitignore from dumux to the new module + dumux_gitignore_file = os.path.join(script_path, "dumux/.gitignore") + shutil.copy(dumux_gitignore_file, module_path) + # delete unnecessary directories to keep the extracted module clean shutil.rmtree('dune') shutil.rmtree('src') # set CMakeLists.txt for each directory generate_cmake_lists_txt_file(module_path) - print("The required header files are extracted and CMakeLists are configured.") - print("=============================================================================") + print("The required header files are extracted" + " and CMakeLists are configured.") + print("==================================================================") # create README file os.remove("README") @@ -335,7 +396,8 @@ duneproject will be run now. The new module should NOT depend on the module in { readme_source_str_lists = [] for source in all_sources: readme_source_str_lists.append("* `" + source + "`,\n") - readme_str_args = [module_dir, ''.join(readme_dir_str_lists), ''.join(readme_source_str_lists)] + readme_str_args = [module_dir, ''.join(readme_dir_str_lists), + ''.join(readme_source_str_lists)] readme_file.write("""This file has been created automatically. Please adapt it to your needs. \n ===============================\n @@ -351,38 +413,85 @@ module by using `dunecontrol`. For building and running the executables, please go to the build folders corresponding to the sources listed above.\n """.format(*readme_str_args)) - version_script_path = os.path.join(script_path, "dumux/bin/util/getusedversions.py") - install_script_path = os.path.join(script_path, "dumux/bin/util/makeinstallscript.py") + version_script_path = os.path.join(script_path, + "dumux/bin/util/getusedversions.py") + install_script_path = os.path.join(script_path, + "dumux/bin/util/makeinstallscript.py") os.chdir(script_path) - if query_yes_no("\nWrite detailed version information (folder/branch/commits/dates) into README.md?"): - print("Looking for the dune modules in path :" + str(script_path) + "...") - readme_file.write("===============================\n" + "## Version Information\n") - versions = getUsedVersions([dep['folder'] for dep in getDependencies(module_full_path)], True) + # ask user if to write version information into README + if query_yes_no("\nWrite detailed version information" + " (folder/branch/commits/dates) into README.md?"): + print("Looking for the dune modules in path :" + str(script_path) + + "...") + readme_file.write("===============================\n" + + "## Version Information\n") + versions = getPersistentVersions( + [dep['folder'] for dep in getDependencies(module_full_path)], True) print("Writing version information into README.md ...") sys.stdout = readme_file printVersionTable(versions) sys.stdout = orig_stdout - readme_file.close() + print("Automatic generation of README.md file is complete.") - print("=============================================================================") + print("==================================================================") + # ask user if to generate an install script install_script_name = 'install_' + module_name + '.sh' - if query_yes_no("\nGenerate install script " + install_script_name + "in your new module " + module_name + "?"): - os.system("python3 " + install_script_path + " -p" + os.path.join(script_path,module_name) + " -i -s " + module_name) - shutil.move(install_script_name, os.path.join(module_path, install_script_name)) - - # output guidence for users - print("\n"+"*"*80+"\n"+ - """The extracted module is contained in the subfolder \"{0}\". - You can build it using \"dunecontrol ... —only= {0} all\". - BEFORE building, you can add the module to dumux-pub by something like: - (Rename module name if it does not match the AuthorLastNameYearx scheme - and commit it to the Git repository dumux-pub) - git clone https://git.iws.uni-stuttgart.de/dumux-pub/AuthorLastNameYearx.git - sed -i '/Module:/c\Module: AuthorLastNameYearx' {0} /dune.module - mv {0} /* dumux-pub/AuthorLastNameYearx/. - cd AuthorLastNameYearx - git commit -a - git push""".format(module_name)+ - "\n"+"*"*80) - + if query_yes_no("\nGenerate install script " + install_script_name + + " in your new module " + module_name + "?"): + remoteurl = get_remote_url(module_path) + print("Write instructions for install script into README.md file...") + readme_file.write("""=============================== +## Installation + The easiest way of installation is to use the install script `{0}` + provided in this repository. + Using `wget`, you can simply install all dependent modules by typing: + + ```sh + wget {1}/{0}.sh + chmod u+x {0} + ./{0} + ``` + + This will create a sub-folder `DUMUX`, clone all modules into it. + +""".format(*[install_script_name, remoteurl])) + readme_file.close() + run_from_mod = callFromPath(module_path)(runCommand) + try: + run_from_mod('git init') + run_from_mod('git add .') + run_from_mod('git commit -m "Initial commit"') + run_from_mod('git remote add origin {}'.format(remoteurl)) + run_from_mod('git push -u origin master') + + makeInstallScript(os.path.join(script_path, module_name), + ignoreUntracked=True) + shutil.move(install_script_name, + os.path.join(module_name, install_script_name)) + run_from_mod('git add .') + run_from_mod('git commit -m "Create Install script"') + run_from_mod('git push -u origin master') + + print("\n" + "*"*80 + "\n" + """Congratulations, the install script {}.sh has been succesfully generated, + You can make further changes to suit your needs, + commit and push to the remote repository""" + .format(module_name) + "\n" + "*"*80) + except Exception: + sys.exit("""Automatically generate install script {}.sh failed. +To create the script, use the "makeinstallscript.py" script in the same folder, +run python3 makeinstallscript.py --help for more detailed information""" + .format(module_name)) + + # in case user want to do generate install.sh manuelly + else: + # output guidence for users + print("\n" + "*"*80 + "\n" + + """The extracted module is contained in the subfolder \"{0}\". + You can configure it using \"dunecontrol ... —only= {0} all\". + Make sure you have an empty remote repository. + To create an install script {0}.sh, + use the "makeinstallscript.py" script in the same folder, + run python3 makeinstallscript.py --help for more information""" + .format(module_name) + "\n" + "*"*80) diff --git a/bin/util/extractmodulepart.sh b/bin/extractmodule/extractmodulepart.sh similarity index 100% rename from bin/util/extractmodulepart.sh rename to bin/extractmodule/extractmodulepart.sh -- GitLab From 623b4213cf1b2b8f4438486f741828ed31cc5f79 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 6 Jun 2021 21:39:30 +0200 Subject: [PATCH 07/47] [bin] Improve extraction script --- bin/extractmodule/extract_as_new_module.py | 743 ++++++++++----------- bin/extractmodule/util.py | 22 +- 2 files changed, 377 insertions(+), 388 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 99e4d3d9a6..5b323f76be 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -11,10 +11,12 @@ import argparse import shutil from distutils.dir_util import copy_tree import re -import threading +import multiprocessing as mp import fnmatch +import itertools +from functools import partial from util import getPersistentVersions -from util import printVersionTable +from util import versionTable from makeinstallscript import makeInstallScript try: path = os.path.split(os.path.abspath(__file__))[0] @@ -24,196 +26,40 @@ try: except Exception: sys.exit('Could not import common modul or getModuleInfo') -# set script parameters -epilog = ''' ------------------------------------------------------------ -The script has to be called one level above module_dir. -At least one of the subfolders (FOLDER_1 [FOLDER_2 ...]) has -to contain a source file *.cc of an executable for which -you would like to timber a table in dumux-pub.) -''' -parser = argparse.ArgumentParser( - prog='extractmodulepart', - usage="./extractmodulepart module_dir SUBFOLDER_1 [SUBFOLDER_2 ...]", - description='This script extracts a subfolder of a DUNE module', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=epilog) -parser.add_argument('module_dir', - help='Dune module from which the subfolder is extracted') -parser.add_argument('subfolder', - nargs='+', - help='subfolder(s) of module_dir that you want to extract') -args = vars(parser.parse_args()) + +# return the list of included headers including the header itself +def add_headers_recursively(header_path, curret_headers, module_path): + hh = [] + if os.path.exists(header_path): + if header_path not in curret_headers: + hh.append(header_path) + hh += search_headers(header_path, module_path) + return hh # function to search matching header files -def search_headers(c_file): - with open(c_file, 'r') as f: - content = f.read() +def search_headers(source_file, module_path): + headers = [] - header_in_bracket = re.findall(r'#include\s+<(.+?)>', content) - header_in_quotation = re.findall(r'#include\s+"(.+?)"', content) + with open(source_file, 'r') as f: + content = f.read() + header_in_bracket = re.findall(r'#include\s+<(.+?)>', content) + header_in_quotation = re.findall(r'#include\s+"(.+?)"', content) # search for includes relative to the module path for header in header_in_bracket + header_in_quotation: - header_with_path = os.path.join(module_full_path, header) - if not os.path.exists(header_with_path): - continue - if header_with_path not in all_headers: - all_headers.append(header_with_path) - thread_ = threading.Thread(target=search_headers, - args=(header_with_path, )) - thread_.start() + header_path = os.path.join(module_path, header) + headers += add_headers_recursively(header_path, headers, module_path) # search for includes relative to the path of the including file # only allow quoted includes for this for header in header_in_quotation: if header == "config.h": continue - header_dir_name = os.path.dirname(c_file) - header_with_path = os.path.join(header_dir_name, header) - if not os.path.exists(header_with_path): - continue - if header_with_path not in all_headers: - all_headers.append(header_with_path) - thread_ = threading.Thread(target=search_headers, - args=(header_with_path, )) - thread_.start() - - -# functions to find the file with specific name in a direcotry -def find_files_with_name(name, path): - result = [] - for root, dirs, files in os.walk(path): - if name in files: - result.append(os.path.join(root, name)) - return result - - -# functions to find the file with specific pattern in a direcotry -def find_files_with_pattern(pattern, path): - result = [] - for root, dirs, files in os.walk(path): - for name in files: - if fnmatch.fnmatch(name, pattern): - result.append(os.path.join(root, name)) - return result - - -# functions to configure CMakeLists.txt files -def generate_new_content(lines, pattern, replace_str): - index = lines.find(pattern) - if index != -1: - content = lines[0: index] + "\n" - if replace_str != "": - content += replace_str + "\n" - flag = True - while flag: - index = lines.find(")", index) - if index != -1: - lines = lines[index + 1:] - else: - break - - index = lines.find(pattern) - if index != -1: - content += lines[0: index] + "\n" - else: - content += lines + "\n" - flag = False - else: - if replace_str == "": - content = lines - else: - if pattern == "add_subdirectory(": - content = replace_str + "\n" + lines - else: - content = lines + "\n" + replace_str - return content - - -def generate_subdirectory_content(dirs): - content = "" - for name in dirs: - content += "add_subdirectory(" + name + ")" + "\n" - return content - - -def generate_install_content(header_files, destination): - if len(header_files) == 0: - return "" - content = "install(FILES" + "\n" - for header_file in header_files: - content += " " + header_file + "\n" - content += ("DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}" - + destination + ")" + "\n") - return content - - -def generate_content(cmake_list_txt_file, dirs, header_files, destination): - subdirectory_content = generate_subdirectory_content(dirs) - install_content = generate_install_content(header_files, destination) - - if os.path.exists(cmake_list_txt_file): - with open(cmake_list_txt_file, "r", encoding="utf-8") as f: - content = "".join(f.readlines()).strip() - f.close() - content = generate_new_content(content, "add_subdirectory(", - subdirectory_content) - return generate_new_content(content, "install(FILE", install_content) - else: - if subdirectory_content == "": - return install_content - else: - return subdirectory_content + "\n" + install_content - - -def generate_cmake_lists_txt(cmake_list_txt_file, dirs, - header_files, destination): - content = generate_content(cmake_list_txt_file, dirs, - header_files, destination) - - with open(cmake_list_txt_file, "w", encoding="utf-8") as f: - if content != "": - f.write(content) - - -def check_dir(root_dir): - if not os.path.exists(root_dir): - print("root path" + str(root_dir) + "is not exist!") - return False - if not os.path.isdir(root_dir): - print("root path" + str(root_dir) + "is not dir!") - return False - return True - - -def check_str(root_dir): - if root_dir is None or root_dir.strip() == "": - return None - root_dir = root_dir.strip() - if root_dir.endswith(os.sep): - root_dir = root_dir[:-1] - return root_dir - - -def generate_cmake_lists_txt_file(root_dir): - root_dir = check_str(root_dir) - if root_dir is None: - return - if not check_dir(root_dir): - return - drop_len = len(root_dir) - for parent, dirs, files in os.walk(root_dir): - destination = parent[drop_len:] - header_files = [] - cmake_list_txt_file = os.path.join(parent, "CMakeLists.txt") - for name in files: - for suffix in [".h", ".hh"]: - if name.endswith(suffix): - header_files.append(name) - generate_cmake_lists_txt(cmake_list_txt_file, dirs, - header_files, destination) + header_path = os.path.join(os.path.dirname(source_file), header) + headers += add_headers_recursively(header_path, headers, module_path) + + return headers # function asking user to answer yes or no question @@ -222,13 +68,13 @@ def query_yes_no(question, default="yes"): if default is None: prompt = " [y/n] " elif default == "yes": - prompt = " [y/N] " - elif default == "no": prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " else: raise ValueError("invalid default answer: '%s'" % default) while True: - sys.stdout.write(question + prompt) + sys.stdout.write("\n\n" + question + prompt) choice = input().lower() if default is not None and choice == "": return valid[default] @@ -239,15 +85,15 @@ def query_yes_no(question, default="yes"): "(or 'y' or 'n').\n") -# funtion to get correct git remote url and make sure it is empty +# query for git remote url and make sure it is not empty def get_remote_url(repo_path): run_from_mod = callFromPath(repo_path)(runCommand) while True: if query_yes_no("Do you own a subfolder in dumux-pub?"): nameyearx = input("Type below the information of" - " AuthorLastNameYearx (e.g. luggi2020b)\n") + " AuthorLastNameYearx (e.g. Luigi2020b)\n") remoteurl = 'https://git.iws.uni-stuttgart.de'\ - '/dumux-pub/{}.git'.format(nameyearx) + '/dumux-pub/{}.git'.format(nameyearx.lower()) else: remoteurl = input("Provide URL of your remote repository:\n") check_remote_repo = run_from_mod('git ls-remote {}'.format(remoteurl)) @@ -259,205 +105,350 @@ def get_remote_url(repo_path): sys.stdout.write("ERROR: The remote reposity is not empty!.\n") -if __name__ == "__main__": +def extract_sources_files(module_dir, subfolders): + def find_files_with_pattern(pattern, path): + result = [] + for root, dirs, files in os.walk(path): + for name in files: + if fnmatch.fnmatch(name, pattern): + result.append(os.path.join(root, name)) + return result + + sources = [] + for folder in subfolders: + folder_path = os.path.join(module_dir, folder) + if not os.path.isdir(folder_path): + raise NameError(f"Subfolder '{folder}' is not a subfolder of '{module_dir}'") + sources += find_files_with_pattern("*.cc", os.path.abspath(folder_path)) + return sources + + +def check_module(module_name): + try: + with open(f"{module_name}/dune.module") as module_file: + content = module_file.read() + if f"Module: {module_name}" not in content: + raise Exception(f"Invalid dune.module in {module_name}") + except OSError: + print("Could not find new Dune module. Aborting") + raise + + +################################################################### +# Some general information for users of the script +################################################################### +def info_explanations(module_dir, module_path, subfolders, source_files): + return f""" +This script will extract the following subfolders of +the module '{module_dir}': +{os.linesep.join([f" - {os.path.relpath(f, module_path)}" for f in subfolders])} + +and all headers contained in '{module_dir}' +tha are required to build the exectutables from the sources: +{os.linesep.join([f" - {s}" for s in source_files])} + +The extracted files are copied into a new DUNE module +retaining the directory structure. Required files for +creating a working DUNE module (like CMakeLists.txt) +will be created and/or updated. + +The script 'duneproject' will run now. Please answer all +upcoming queries to the best of your knowledge. + +Important: the new module should NOT depend on the module '{module_dir}' +""" + + +################################################################### +# Main part of README.md +################################################################### +def info_readme_main(module_dir, subfolders, source_files): + subfolders_str = ''.join([f"* `{d}`\n" for d in subfolders]) + sources_str = ''.join([f"* `{s}`\n" for s in source_files]) + return f""" +This file has been created automatically. Please adapt it to your needs. + +## Content + +The content of this DUNE module has been extracted from the module `{module_dir}`. +In particular, the following subfolders of `{module_dir}` have been extracted: +{subfolders_str} +Additionally, all headers in `{module_dir}` that are required to build the +executables from the sources +{sources_str} +have been extracted. You can configure the module just like any other DUNE +module by using `dunecontrol`. For building and running the executables, +please go to the build folders corresponding to the sources listed above.\n +""" + + +################################################################### +# Installation part of README.md +################################################################### +def info_readme_installation(remoteurl, install_script_name): + return f""" + +## Installation + +The easiest way of installation is to use the install script `{install_script_name}` +provided in this repository. +Using `wget`, you can simply install all dependent modules by typing: + +```sh +wget {remoteurl}/{install_script_name}.sh +chmod u+x {install_script_name} +./{install_script_name} +``` + +This will create a sub-folder `DUMUX`, clone all modules into it. + +""" + - module_dir = args['module_dir'] - subfolders = args['subfolder'] +################################################################### +# Infos on how to create the install script manually +################################################################### +def info_make_install(new_module_name): + return f""" +======================================================================== - # if module_dir contains a slash as last character, delete it - if module_dir.endswith('/'): - module_dir = module_dir[:-1] +The extracted module is contained in the subfolder '{new_module_name}'. +You can configure it with +./dune-common/bin/dunecontrol --opts=dumux/cmake.opts —only={new_module_name} all +To create an install script use the "makeinstallscript.py" script in the same folder. +You can run 'python3 makeinstallscript.py --help' for more information +""" + + +if __name__ == "__main__": + + # set script parameters + epilog = ''' + ----------------------------------------------------------- + The script has to be called one level above module_dir. + At least one of the subfolders (FOLDER_1 [FOLDER_2 ...]) has + to contain a source file *.cc of an executable for which + you would like to timber a table in dumux-pub.) + ----------------------------------------------------------- + + Example usage: + ./dumux/bin/extractmodule/extract_as_new_module.py dumux-fracture appl test + + (extracts the subfolders appl and test from the module dumux-fracture) + + ''' + parser = argparse.ArgumentParser( + prog='extract_as_new_module.py', + usage='./dumux/bin/extractmodule/extract_as_new_module.py' + ' module_dir SUBFOLDER_1 [SUBFOLDER_2 ...]', + description='This script extracts subfolders of a given DUNE module' + ' into a new DUNE module.', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=epilog) + parser.add_argument( + 'module_dir', + help='Dune module from which the subfolder is extracted' + ) + parser.add_argument( + 'subfolder', nargs='+', + help='subfolder(s) of module_dir that you want to extract' + ) + args = vars(parser.parse_args()) + + # if module_dir ends with slash(es) remove it/them + module_dir = args['module_dir'].strip(os.sep) + module_path = os.path.abspath(module_dir) + # make unique set of subfolders + subfolders = list(set(args['subfolder'])) # check if we are above module_dir - if not (os.path.isdir(module_dir)): - print("ERROR: you need to run the script one level above the folder " - + module_dir + ".") - print("Run \""+os.path.basename(__file__)+" --help\" for details.") - exit(1) + if not os.path.isdir(module_dir): + sys.exit("ERROR: You need to run the script" + f"one level above the folder {module_dir}.\n" + f"Run \"{os.path.basename(__file__)} --help\" for details.") # determine all source files in the paths passed as arguments - script_path = os.getcwd() - os.chdir(module_dir) - module_full_path = os.getcwd() - all_sources = [] - all_sources_with_path = [] - all_directories = [] - for dir_path in subfolders: - if dir_path.startswith(module_dir): - # the "+1" is required to also remove the "/" - stripped_path = dir_path[len(module_dir)+1:] - else: - stripped_path = dir_path - full_path = os.path.join(module_full_path, stripped_path) - directories = " " + stripped_path - all_directories.append(stripped_path) - os.chdir(full_path) - for file in find_files_with_pattern("*.cc", full_path): - print(file) - sources = os.path.relpath(file, module_full_path) - print(sources) - all_sources.append(sources) - sourceswithpath = os.path.join(module_full_path, sources) - all_sources_with_path.append(sourceswithpath) - os.chdir(module_full_path) - os.chdir("..") # back to the script folder + source_files = extract_sources_files(module_dir, subfolders) # check if sources have been obtained - if (all_sources == []): - print("ERROR: no source files *.cc found in the directories " - + " ".join([str(x) for x in subfolders]) + ".") - print("Be sure to provide a list of paths as arguments.") - print("Run \""+os.path.basename(__file__)+" --help\" for details.") - exit(1) + if not source_files: + sys.exit( + "ERROR: No source files *.cc found in the subfolders: " + + ", ".join([str(x) for x in subfolders]) + ".\n" + "Be sure to provide a list of paths as arguments.\n" + f"Run '{os.path.basename(__file__)} --help' for details." + ) # try to find the duneproject script dune_project = shutil.which('duneproject', path="dune-common/bin") - if (dune_project is None): - print("ERROR: Could not find duneproject.") - print("Be sure to have duneproject in dune-common/bin") - exit(1) - else: - print(dune_project) + if dune_project is None: + sys.exit( + "ERROR: Could not find duneproject.\n" + "Make sure to have duneproject in dune-common/bin" + ) # give explanations - print("\n""This script will\n" - "- extract the following sub-folders of " + module_dir + ":\n") - for dir_path in all_directories: - print(" " + dir_path + ",") - print("\n"" and all headers in " + module_dir - + " that are required to build the\n" - " executables from the sources\n") - for source in all_sources: - print(" " + source + ",") - print(""" -This script copies extracted files into a freshly created DUNE module -retaining the directory structure -and updates/creates required files like CMakeLists.txt. -duneproject will run now. -The new module should NOT depend on the module in {0}.\n\n""" - .format(module_dir)) - + print(info_explanations( + module_dir, module_path, subfolders, source_files + )) input("Read the above and press [Enter] to proceed...") # run duneproject - old_ls = os.listdir() subprocess.call([dune_project]) - new_ls = os.listdir() - # determine the new module/directory name - module_name = (set(new_ls) - set(old_ls)).pop() - if (module_name == ""): - print("ERROR: could not find new module. Aborting.") - exit(1) - else: - print() - print(os.path.basename(__file__) + ": Found new module " + module_name) - print("Determining required headers...") - os.chdir(module_name) - module_path = os.getcwd() - - # extract headers, provide some output and copy everything to new module - all_headers = [] - print("The following header files are extracted: ") - for source in all_sources_with_path: - search_headers(source) - for header in all_headers: - print(header) - dir_path = os.path.dirname(os.path.realpath(header))\ - .replace(module_dir, module_name, 1) - os.makedirs(dir_path, exist_ok=True) - shutil.copy(header, dir_path) - source_dir = os.path.dirname(source) - source_path = source_dir.replace(module_dir, module_name, 1) - copy_tree(source_dir, source_path) + # find the created folder + # as the one with the most recent modification time + new_module_name = max( + [d for d in os.listdir() if os.path.isdir(d)], + key=os.path.getmtime + ) + + # verify it's really a Dune module + check_module(new_module_name) + print( + f"Found new module {new_module_name}\n" + "Copying source files..." + ) + + # get the base path of the new module + new_module_path = module_path.replace(module_dir, new_module_name, 1) + + # copy the source tree + # copy all base folders complete, then delete the unnecessary ones + base_folders = list(set([ + os.path.relpath(s, module_path).split(os.path.sep)[0] for s in source_files + ])) + for b in base_folders: + path_in_old_module = os.path.join(module_path, b) + path_in_new_module = os.path.join(new_module_path, b) + copy_tree(path_in_old_module, path_in_new_module) + + # add base folders in project-level CMakeLists.txt + with open(os.path.join(new_module_path, "CMakeLists.txt"), "r") as cml: + section = 0 + content = [] + for line in cml.readlines(): + if line.startswith("add_subdirectory"): + section = 1 if section in [0, 1] else 2 + elif section == 1: + section = 2 + for b in base_folders: + content.append(f"add_subdirectory({b})") + content.append(line) + + with open(os.path.join(new_module_path, "CMakeLists.txt"), "w") as cml: + for line in content: + cml.write(line) + + # go through source tree and remove unnecessary directories + new_source_files = [ + s.replace(module_dir, new_module_name, 1) for s in source_files + ] + for b in base_folders: + def matching_path(path, list_of_paths): + for p in list_of_paths: + if path in p: + return True + return False + + for path, dirs, files in os.walk(os.path.join(new_module_path, b)): + for d in dirs: + dir_path = os.path.join(path, d) + keep = matching_path(dir_path, new_source_files) + if not keep: + rel_dir_path = os.path.relpath(dir_path, new_module_path) + answer_is_yes = query_yes_no( + f"{rel_dir_path} does not contain source files." + " Copy to new module?", + default="no" + ) + if not answer_is_yes: + # remove copy of directory + shutil.rmtree(dir_path) + # remove entry from CMakeLists.txt + cml_path = os.path.join(module_path, "CMakeLists.txt") + with open(cml_path, "r+") as cml: + content = cml.read() + cml.seek(0) + content = content.replace( + "add_subdirectory({d})\n", "" + ) + cml.write(content) + cml.truncate() + + # make os.walk know about removed folders + dirs.remove(d) + + # search for all header (in parallel) + with mp.Pool() as p: + headers = itertools.chain.from_iterable(p.map( + partial(search_headers, module_path=module_path), + source_files + )) + + # make unique + headers = list(set(headers)) + + # copy headers to the new module + for header in headers: + header_dir = os.path.dirname(os.path.realpath(header)) + path_in_new_module = header_dir.replace(module_dir, new_module_name, 1) + os.makedirs(path_in_new_module, exist_ok=True) + shutil.copy(header, path_in_new_module) # copy .gitignore from dumux to the new module - dumux_gitignore_file = os.path.join(script_path, "dumux/.gitignore") - shutil.copy(dumux_gitignore_file, module_path) + dumux_gitignore_file = "dumux/.gitignore" + shutil.copy(dumux_gitignore_file, new_module_path) # delete unnecessary directories to keep the extracted module clean - shutil.rmtree('dune') - shutil.rmtree('src') - - # set CMakeLists.txt for each directory - generate_cmake_lists_txt_file(module_path) - print("The required header files are extracted" - " and CMakeLists are configured.") - print("==================================================================") + if "dune" not in subfolders: + shutil.rmtree(os.path.join(new_module_path, 'dune')) + if "src" not in subfolders: + shutil.rmtree(os.path.join(new_module_path, 'src')) + with open(os.path.join(new_module_path, "CMakeLists.txt"), "r+") as cml: + content = cml.read() + cml.seek(0) + if "dune" not in subfolders: + content = content.replace("add_subdirectory(dune)\n", "") + if "src" not in subfolders: + content = content.replace("add_subdirectory(src)\n", "") + cml.write(content) + cml.truncate() # create README file - os.remove("README") - orig_stdout = sys.stdout - readme_file = open("README.md", "w+") - - readme_dir_str_lists = [] - for dir_path in all_directories: - readme_dir_str_lists.append("* `" + dir_path + "`,\n") - readme_source_str_lists = [] - for source in all_sources: - readme_source_str_lists.append("* `" + source + "`,\n") - readme_str_args = [module_dir, ''.join(readme_dir_str_lists), - ''.join(readme_source_str_lists)] - - readme_file.write("""This file has been created automatically. Please adapt it to your needs. \n -===============================\n -## Content\n -"The content of this DUNE module was extracted from the module `{0}`." -In particular, the following subfolders of `{0}` have been extracted: -{1} -Additionally, all headers in `{0}` that are required to build the -executables from the sources -{2} -have been extracted. You can configure the module just like any other DUNE -module by using `dunecontrol`. For building and running the executables, -please go to the build folders corresponding to the sources listed above.\n -""".format(*readme_str_args)) - - version_script_path = os.path.join(script_path, - "dumux/bin/util/getusedversions.py") - install_script_path = os.path.join(script_path, - "dumux/bin/util/makeinstallscript.py") - os.chdir(script_path) - # ask user if to write version information into README - if query_yes_no("\nWrite detailed version information" + os.remove(os.path.join(new_module_path, 'README')) + readme_path = os.path.join(new_module_path, "README.md") + with open(readme_path, "w") as readme_file: + readme_file.write( + info_readme_main(module_dir, subfolders, source_files) + ) + + # ask user if to write version information into README.md + if query_yes_no("Write detailed version information" " (folder/branch/commits/dates) into README.md?"): - print("Looking for the dune modules in path :" + str(script_path) + print("Looking for the dune modules in path: " + + os.path.abspath(".") + "...") - readme_file.write("===============================\n" - + "## Version Information\n") - versions = getPersistentVersions( - [dep['folder'] for dep in getDependencies(module_full_path)], True) - print("Writing version information into README.md ...") - sys.stdout = readme_file - printVersionTable(versions) - sys.stdout = orig_stdout - - print("Automatic generation of README.md file is complete.") - print("==================================================================") - - # ask user if to generate an install script - install_script_name = 'install_' + module_name + '.sh' - if query_yes_no("\nGenerate install script " + install_script_name - + " in your new module " + module_name + "?"): - remoteurl = get_remote_url(module_path) - print("Write instructions for install script into README.md file...") - readme_file.write("""=============================== -## Installation - The easiest way of installation is to use the install script `{0}` - provided in this repository. - Using `wget`, you can simply install all dependent modules by typing: - - ```sh - wget {1}/{0}.sh - chmod u+x {0} - ./{0} - ``` - - This will create a sub-folder `DUMUX`, clone all modules into it. -""".format(*[install_script_name, remoteurl])) - readme_file.close() - run_from_mod = callFromPath(module_path)(runCommand) + versions = getPersistentVersions( + [dep['folder'] for dep in getDependencies(module_path)], True + ) + + # append version information + with open(readme_path, "a") as readme_file: + readme_file.write( + "\n## Version Information\n\n" + + versionTable(versions) + ) + + # if there is a remote repository available + # we can directly push the source code and + # also create a "one-click" installation script + install_script_name = 'install_' + new_module_name + '.sh' + if query_yes_no("Do you have an empty remote repository to push the code to (recommended)?"): + remoteurl = get_remote_url(new_module_path) + + run_from_mod = callFromPath(new_module_path)(runCommand) try: run_from_mod('git init') run_from_mod('git add .') @@ -465,33 +456,27 @@ please go to the build folders corresponding to the sources listed above.\n run_from_mod('git remote add origin {}'.format(remoteurl)) run_from_mod('git push -u origin master') - makeInstallScript(os.path.join(script_path, module_name), - ignoreUntracked=True) + # create an installation script + makeInstallScript(new_module_path, ignoreUntracked=True) shutil.move(install_script_name, - os.path.join(module_name, install_script_name)) + os.path.join(new_module_path, install_script_name)) + + # append install information into readme + with open(readme_path, "a") as readme_file: + readme_file.write(info_readme_installation(remoteurl, install_script_name)) + + # add to version control run_from_mod('git add .') run_from_mod('git commit -m "Create Install script"') run_from_mod('git push -u origin master') - print("\n" + "*"*80 + "\n" - """Congratulations, the install script {}.sh has been succesfully generated, - You can make further changes to suit your needs, - commit and push to the remote repository""" - .format(module_name) + "\n" + "*"*80) except Exception: - sys.exit("""Automatically generate install script {}.sh failed. -To create the script, use the "makeinstallscript.py" script in the same folder, -run python3 makeinstallscript.py --help for more detailed information""" - .format(module_name)) + sys.exit( + f"Automatically generate install script {install_script_name} failed." + "\nTo create the script, use the 'makeinstallscript.py' script in the same folder." + "\nRun 'python3 makeinstallscript.py --help' for more detailed information" + ) - # in case user want to do generate install.sh manuelly + # output guidance for users to create install script manually else: - # output guidence for users - print("\n" + "*"*80 + "\n" + - """The extracted module is contained in the subfolder \"{0}\". - You can configure it using \"dunecontrol ... —only= {0} all\". - Make sure you have an empty remote repository. - To create an install script {0}.sh, - use the "makeinstallscript.py" script in the same folder, - run python3 makeinstallscript.py --help for more information""" - .format(module_name) + "\n" + "*"*80) + print(info_make_install(new_module_name)) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index 1cf2da7a5d..0394957063 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -123,13 +123,17 @@ def getPatches(persistentVersions): return result +def versionTable(versions): + return "| {:<50} | {:<50} | {:<50} | {:<30} |\n".format( + 'module folder', 'branch', 'commit hash', 'commit date' + ) + "| {:<50} | {:<50} | {:<50} | {:<30} |\n".format( + '-'*50, '-'*50, '-'*50, '-'*30 + ) + "\n".join( + ["| {:<50} | {:<50} | {:<50} | {:<30} |".format( + folder, versionInfo['branch'], versionInfo['revision'], versionInfo['date'] + ) for folder, versionInfo in versions.items()] + ) + "\n" + + def printVersionTable(versions): - print("\t| {:^50} | {:^50} | {:^50} | {:^30} |" - .format('module folder', 'branch', 'commit hash', 'commit date')) - print("\t" + 193*'-') - for folder, versionInfo in versions.items(): - print("\t| {:^50} | {:^50} | {:^50} | {:^30} |" - .format(folder, - versionInfo['branch'], - versionInfo['revision'], - versionInfo['date'])) + print(versionTable(versions=versions)) -- GitLab From a8cba3f1582e15fcf35720c93613278a79344a93 Mon Sep 17 00:00:00 2001 From: hanchuan <> Date: Wed, 9 Jun 2021 18:02:12 +0200 Subject: [PATCH 08/47] [bin][extractmod] Fix extract script --- bin/extractmodule/extract_as_new_module.py | 94 +++++++++++----------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 5b323f76be..6ca8ecdea0 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -28,28 +28,24 @@ except Exception: # return the list of included headers including the header itself -def add_headers_recursively(header_path, curret_headers, module_path): - hh = [] +def add_headers_recursively(header_path, current_headers, module_path): if os.path.exists(header_path): - if header_path not in curret_headers: - hh.append(header_path) - hh += search_headers(header_path, module_path) - return hh + if header_path not in current_headers: + current_headers.append(header_path) + search_headers(header_path, current_headers, module_path) # function to search matching header files -def search_headers(source_file, module_path): - headers = [] - +def search_headers(source_file, headers, module_path): with open(source_file, 'r') as f: content = f.read() header_in_bracket = re.findall(r'#include\s+<(.+?)>', content) header_in_quotation = re.findall(r'#include\s+"(.+?)"', content) # search for includes relative to the module path - for header in header_in_bracket + header_in_quotation: + for header in header_in_bracket: header_path = os.path.join(module_path, header) - headers += add_headers_recursively(header_path, headers, module_path) + add_headers_recursively(header_path, headers, module_path) # search for includes relative to the path of the including file # only allow quoted includes for this @@ -57,7 +53,7 @@ def search_headers(source_file, module_path): if header == "config.h": continue header_path = os.path.join(os.path.dirname(source_file), header) - headers += add_headers_recursively(header_path, headers, module_path) + add_headers_recursively(header_path, headers, module_path) return headers @@ -185,19 +181,18 @@ please go to the build folders corresponding to the sources listed above.\n ################################################################### # Installation part of README.md ################################################################### -def info_readme_installation(remoteurl, install_script_name): +def info_readme_installation(remoteurl, install_script_name, new_module_name): return f""" ## Installation The easiest way of installation is to use the install script `{install_script_name}` provided in this repository. -Using `wget`, you can simply install all dependent modules by typing: +You can simply clone the remote and install all dependent modules by typing: ```sh -wget {remoteurl}/{install_script_name}.sh -chmod u+x {install_script_name} -./{install_script_name} +git clone {remoteurl} +./{new_module_name}/{install_script_name} ``` This will create a sub-folder `DUMUX`, clone all modules into it. @@ -382,7 +377,7 @@ if __name__ == "__main__": # search for all header (in parallel) with mp.Pool() as p: headers = itertools.chain.from_iterable(p.map( - partial(search_headers, module_path=module_path), + partial(search_headers, module_path=module_path, headers=[]), source_files )) @@ -445,38 +440,43 @@ if __name__ == "__main__": # we can directly push the source code and # also create a "one-click" installation script install_script_name = 'install_' + new_module_name + '.sh' + try: + makeInstallScript(new_module_path, ignoreUntracked=True, skipFolders=new_module_name, + suppressHints=True, topFolderName=None) + except Exception: + sys.exit( + f"Automatically generate install script {install_script_name} failed." + "\nTo create the script, use the 'makeinstallscript.py' script in the same folder." + "\nRun 'python3 makeinstallscript.py --help' for more detailed information" + ) + + shutil.move(install_script_name, + os.path.join(new_module_path, install_script_name)) + print(info_make_install(new_module_name)) + run_from_mod = callFromPath(new_module_path)(runCommand) + if query_yes_no("Do you have an empty remote repository to push the code to (recommended)?"): remoteurl = get_remote_url(new_module_path) - run_from_mod = callFromPath(new_module_path)(runCommand) - try: - run_from_mod('git init') - run_from_mod('git add .') - run_from_mod('git commit -m "Initial commit"') - run_from_mod('git remote add origin {}'.format(remoteurl)) - run_from_mod('git push -u origin master') - - # create an installation script - makeInstallScript(new_module_path, ignoreUntracked=True) - shutil.move(install_script_name, - os.path.join(new_module_path, install_script_name)) - - # append install information into readme - with open(readme_path, "a") as readme_file: - readme_file.write(info_readme_installation(remoteurl, install_script_name)) - - # add to version control - run_from_mod('git add .') - run_from_mod('git commit -m "Create Install script"') - run_from_mod('git push -u origin master') - - except Exception: - sys.exit( - f"Automatically generate install script {install_script_name} failed." - "\nTo create the script, use the 'makeinstallscript.py' script in the same folder." - "\nRun 'python3 makeinstallscript.py --help' for more detailed information" - ) + # append install information into readme + with open(readme_path, "a") as readme_file: + readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name)) + run_from_mod('git init') + run_from_mod('git add .') + run_from_mod('git commit -m "Initial commit"') + run_from_mod('git remote add origin {}'.format(remoteurl)) + run_from_mod('git push -u origin master') + + if query_yes_no("Do you also want to create an one-click install script?"): + makeInstallScript(new_module_path, ignoreUntracked=True, suppressHints=True) + print("The one-click install script {} is created in your current folder.".format(install_script_name)) # output guidance for users to create install script manually else: - print(info_make_install(new_module_name)) + remoteurl = "{$remoteurl$} ({$remoteurl$} is the URL for your remote git repository)" + # append install information into readme + with open(readme_path, "a") as readme_file: + readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name)) + run_from_mod('git init') + run_from_mod('git add .') + run_from_mod('git commit -m "Initial commit"') -- GitLab From 075e5638195c0a2ae1c66380f3364a75d7b2dde2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= Date: Tue, 8 Jun 2021 16:47:57 +0200 Subject: [PATCH 09/47] [bin][makeinstall] avoid linter warnings --- bin/extractmodule/makeinstallscript.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 2ed5845081..046238192d 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -37,15 +37,15 @@ def makeInstallScript(path, try: modName = getModuleInfo(modPath, 'Module') except Exception: - sys.exit("\nError: Could not determine module name. Make sure the module\n" - " path is correct and that it contains 'dune.module'.") + sys.exit("\nError: Could not determine module name. Make sure the\n" + " module path is correct and contains 'dune.module'.") if not fileName: instFileName = 'install_' + modName + '.sh' else: instFileName = fileName print("\n-- Creating install script '{}' for module '{}' in folder '{}'" - .format(instFileName, modName, modPath)) + .format(instFileName, modName, modPath)) print("\n-- Determining the dependencies") -- GitLab From 7d06b75b8660a4446340243f8df83b7626114e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= Date: Tue, 8 Jun 2021 16:48:26 +0200 Subject: [PATCH 10/47] [bin][makeinstall] fix rev & branch detection --- bin/extractmodule/util.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index 0394957063..6e45b9a50f 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -59,7 +59,7 @@ def mostRecentCommonCommitWithRemote(modFolderPath, for rev in revList: branches = findBranches(rev) if branches: - return branches[0] + return branches[0], rev raise RuntimeError('Could not find suitable ancestor commit' ' on a branch that matches the given filter') @@ -85,7 +85,7 @@ def getPersistentVersions(modFolderPaths, ignoreUntracked=False): # update remote to make sure we find all upstream commits fetchRepo(result[modFolderPath]['remote'], modFolderPath) - rev = mostRecentCommonCommitWithRemote(modFolderPath) + branch, rev = mostRecentCommonCommitWithRemote(modFolderPath) run = callFromPath(modFolderPath)(runCommand) result[modFolderPath]['revision'] = rev @@ -97,9 +97,7 @@ def getPersistentVersions(modFolderPaths, ignoreUntracked=False): ).strip('\n') # this may return HEAD if we are on some detached HEAD tree - result[modFolderPath]['branch'] = run( - 'git rev-parse --abbrev-ref HEAD' - ).strip('\n') + result[modFolderPath]['branch'] = branch return result -- GitLab From 0fc84022cffe5c407bc43e48680c5ad131678d46 Mon Sep 17 00:00:00 2001 From: hanchuan <> Date: Fri, 11 Jun 2021 10:28:28 +0200 Subject: [PATCH 11/47] [makeinstall] Add the option to skip some folders --- bin/extractmodule/makeinstallscript.py | 43 ++++++++++++++++---------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 046238192d..6ca9003b78 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -27,7 +27,8 @@ def makeInstallScript(path, fileName=None, ignoreUntracked=False, topFolderName='DUMUX', - optsFile=None): + optsFile=None, + skipFolders=None): cwd = os.getcwd() modPath = os.path.abspath(os.path.join(cwd, path)) @@ -59,10 +60,14 @@ def makeInstallScript(path, else: sys.exit("Error: Could not determine module dependencies.") - depNames = [dep['name'] for dep in deps] - depFolders = [dep['folder'] for dep in deps] - depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] - + if not skipFolders: + depNames = [dep['name'] for dep in deps] + depFolders = [dep['folder'] for dep in deps] + depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders] + else: + depNames = [dep['name'] for dep in deps if dep['folder'] != skipFolders] + depFolders = [dep['folder'] for dep in deps if dep['folder'] != skipFolders] + depFolderPaths = [os.path.abspath(os.path.join(modParentPath, d)) for d in depFolders if d != skipFolders] print("\n-- Determining the module versions") try: @@ -277,17 +282,17 @@ def makeInstallScript(path, " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") - - print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" - f" ## Installation\n" - f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" - f" Using `wget`, you can simply install all dependent modules by typing:\n" - f"\n" - f" ```sh\n" - f" wget {versions[modFolder]['remote']}/{instFileName}\n" - f" chmod u+x {instFileName}\n" - f" ./{instFileName}\n" - f" ```\n") + if not skipFolders: + print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" + f" ## Installation\n" + f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" + f" Using `wget`, you can simply install all dependent modules by typing:\n" + f"\n" + f" ```sh\n" + f" wget {versions[modFolder]['remote']}/{instFileName}\n" + f" chmod u+x {instFileName}\n" + f" ./{instFileName}\n" + f" ```\n") if topFolderName: print(f" This will create a sub-folder `{topFolderName}`, clone all modules into it, configure the entire project and build the applications contained in this module.") @@ -326,6 +331,9 @@ if __name__ == '__main__': 'dunecontrol. Note that this file is required to be ' 'contained and committed within the module or its ' 'dependencies.') + parser.add_argument('-s', '--skipfolders', + required=False, default=None, + help='a list of module folders to be skipped') cmdArgs = vars(parser.parse_args()) makeInstallScript( @@ -333,5 +341,6 @@ if __name__ == '__main__': fileName=cmdArgs.get('filename', None), ignoreUntracked=cmdArgs.get('ignoreuntracked', False), topFolderName=cmdArgs.get('topfoldername', None), - optsFile=cmdArgs.get('optsFile', None) + optsFile=cmdArgs.get('optsFile', None), + skipFolders=cmdArgs.get('skipfolders', None) ) -- GitLab From ed9011e103692fd7d56361c0bd46214e130b8b07 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Tue, 15 Jun 2021 10:40:35 +0200 Subject: [PATCH 12/47] [makeinstall] Move new module into the topFolder --- bin/extractmodule/makeinstallscript.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 6ca9003b78..8134ce8792 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -244,6 +244,11 @@ def makeInstallScript(path, .format(depModName) + '\n') installFile.write('cd ..\n\n') + writeCommandWithErrorCheck( + 'mv ../{} .'.format(modFolder), + '-- Error: failed to move the new module to {}'.format(topFolderName) + ) + # write the module clone first in order for the patches to be present if modName in depNames: writeCloneModule(modName, modFolder) -- GitLab From 1ba1d99650f107c5427f9221649b1ecf6459824d Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 16 Jun 2021 16:45:24 +0200 Subject: [PATCH 13/47] [makeinstall] Make shell script executable --- bin/extractmodule/makeinstallscript.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 8134ce8792..b71c55726a 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -6,6 +6,7 @@ import os import sys import argparse +import subprocess from util import getPersistentVersions from util import printVersionTable @@ -282,6 +283,8 @@ def makeInstallScript(path, print("\n-- Successfully created install script file " + instFileName) + subprocess.call(['chmod', 'u+x', instFileName]) # make script executable + if len(patches) > 0: print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" -- GitLab From d2c31c1dbca7bcdd449f9421b0e6b1c9ac8b62d3 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Sun, 27 Jun 2021 23:58:19 +0200 Subject: [PATCH 14/47] [bin] Remove extract module shell script --- bin/extractmodule/extractmodulepart.sh | 398 ------------------------- 1 file changed, 398 deletions(-) delete mode 100755 bin/extractmodule/extractmodulepart.sh diff --git a/bin/extractmodule/extractmodulepart.sh b/bin/extractmodule/extractmodulepart.sh deleted file mode 100755 index 433a79fcd7..0000000000 --- a/bin/extractmodule/extractmodulepart.sh +++ /dev/null @@ -1,398 +0,0 @@ -#! /bin/bash - -# check if help is needed -if test "$1" = "--help" || test "$1" = "-help" \ - || test "$1" = "help" || test "$1" = ""; then - echo "" - echo "USAGE: $0 MODULE_DIR FOLDER_1 [FOLDER_2 ...]" - echo "" - echo "MODULE_DIR is the folder containing the DUNE module from which you" - echo "want to extract. The script has to be called one level above it." - echo "" - echo "The FOLDERs need to indicate subfolders of MODULE_DIR. At least one" - echo "of them has to contain a source file *.cc of an executable for which" - echo "you would like to timber a table in dumux-pub." - exit 0 -fi - -MODULE_DIR=$1 -# if MODULE_DIR contains a slash as last character, delete it -MODULE_DIR=$(echo $MODULE_DIR | sed s'/\/$//') -# check if we are above MODULE_DIR -if (! [ -d $MODULE_DIR ]); then - echo "ERROR: you need to run the script one level above the folder $MODULE_DIR." - echo "Run \"$0 --help\" for details." - exit 1 -fi - -# determine all source files in the paths passed as arguments -cd $MODULE_DIR -MODULE_FULL_PATH=$(pwd) -ALL_SOURCES="" -ALL_DIRECTORIES="" -for DIR_PATH in ${*:2}; do - STRIPPED_PATH=$(sed 's%.*'"$MODULE_DIR"'/%%g' <(echo "$DIR_PATH")) - ALL_DIRECTORIES="$ALL_DIRECTORIES $STRIPPED_PATH" - ALL_SOURCES="$ALL_SOURCES $(find $STRIPPED_PATH -name '*.cc' 2>/dev/null)" -done -cd .. - -# check if sources have been obtained -CONTRACTED="$(echo "$ALL_SOURCES" | tr -d " \t\n\r")" -if test "$CONTRACTED" = ""; then - echo "ERROR: no source files *.cc found in the directories ${*:2}." - echo "Be sure to provide a list of paths as arguments to this script." - echo "Run \"$0 --help\" for details." - exit 1; -fi - -# try to find the duneproject script -if hash duneproject 2>/dev/null; then - DUNE_PROJECT="duneproject" -else - DUNE_PROJECT=$(find . -name 'duneproject') -fi -if test "$DUNE_PROJECT" = ""; then - echo "ERROR: Could not find duneproject." - echo "Be sure to either have duneproject in your search path" - echo "or to run this script from a directory that contains duneproject." - exit 1; -fi - -# give explanations -echo "" -echo "This script will" -echo "- extract the following sub-folders of $MODULE_DIR:" -echo "" -for DIR_PATH in $ALL_DIRECTORIES; do - echo " $DIR_PATH," -done -echo "" -echo " and all headers in $MODULE_DIR that are required to build the" -echo " executables from the sources" -echo "" -for SOURCE in $ALL_SOURCES; do - echo " $SOURCE," -done -echo "" -echo "- copy the extracted files into a freshly created DUNE module, retaining the" -echo " directory structure," -echo "" -echo "- update/create all required files like CMakeLists.txt," -echo "" -echo "- store the versions of all used Dune module" -echo "" -echo "- and extract their modifications as patches." -echo "" -echo "Thus, you receive a fully-working DUNE module containing the subset of" -echo "$MODULE_DIR that is required to run your application." -echo "" -echo "duneproject will be run now. The new module should NOT depend on the" -echo "module in $MODULE_DIR." -echo "" -read -p "Read the above and press [Enter] to proceed..." - -# run duneproject -OLD_LS="$(ls)" -$DUNE_PROJECT -NEW_LS="$(ls)" - -# determine the new module/directory name -DIFF_OUTPUT=$(diff <(echo "$OLD_LS" ) <(echo "$NEW_LS")) -FOUND="0" -MODULE_NAME="" -for WORD in $DIFF_OUTPUT; do - if test "$FOUND" = "1"; then - MODULE_NAME=$WORD - fi - if test "$WORD" = ">"; then - FOUND="1" - fi -done -if test "$MODULE_NAME" = ""; then - echo "ERROR: could not find new module. Aborting." - exit 1 -else - echo "" - echo "$(basename $0): Found new module $MODULE_NAME" -fi - -echo -n "Determining required headers..." -cd $MODULE_DIR - -# initialize lists to hold required headers -LAST_REQUIRED_HEADERS="" -REQUIRED_HEADERS="$ALL_SOURCES" - -while test "$LAST_REQUIRED_HEADERS" != "$REQUIRED_HEADERS"; do - echo -n "." - LAST_REQUIRED_HEADERS="$REQUIRED_HEADERS" - - # remove the file that stores all required headers line by line - rm -f tmp_header_file - - for HEADER in $REQUIRED_HEADERS; do - # if tmp_header_file does not exist, create it - if test "$(ls tmp_header_file 2>/dev/null)" = ""; then - touch tmp_header_file - fi - - # append header name to tmp_header_file - echo "$HEADER" >> tmp_header_file - - # obtain the base name and the directory name of the header that is considered - HEADER_BASE_NAME=$(basename $HEADER) - HEADER_DIR_NAME=$(dirname $HEADER) - - # create a list of all files that are included from the header - sed -i 's/#include/#include /' $HEADER - INCLUDE_LIST=$(tr -s " " < $HEADER | tr -d '><"' | awk '/#include/{print $2}') - sed -i 's/#include /#include/' $HEADER - - # look at each of the included files - for INCLUDED_HEADER in $INCLUDE_LIST; do - # don't include config.h - if test "$INCLUDED_HEADER" = "config.h"; then - continue - fi - - INCLUDED_BASE_NAME=$(basename $INCLUDED_HEADER) - INCLUDED_DIR_NAME=$(dirname $INCLUDED_HEADER) - - # if the header file exists, add it - if test "$(ls $INCLUDED_HEADER 2>/dev/null)" = "$INCLUDED_HEADER"; then - echo "$INCLUDED_HEADER" >> tmp_header_file - continue - fi - - # try the header preceded by its path - INCLUDED_HEADER_WITH_PATH="${HEADER_DIR_NAME}/${INCLUDED_HEADER}" - - # if the header file actually exists, add it - if test "$(ls $INCLUDED_HEADER_WITH_PATH 2>/dev/null)" = "$INCLUDED_HEADER_WITH_PATH"; then - # remove "../" from the path - cd $(dirname $INCLUDED_HEADER_WITH_PATH) - HEADER_FULL_PATH=$(pwd) - HEADER_RELATIVE_PATH=${HEADER_FULL_PATH#$MODULE_FULL_PATH} - HEADER_RELATIVE_PATH=$(echo $HEADER_RELATIVE_PATH | sed 's/^.//') - INCLUDED_HEADER_WITH_PATH="${HEADER_RELATIVE_PATH}/${INCLUDED_BASE_NAME}" - cd $MODULE_FULL_PATH - echo "$INCLUDED_HEADER_WITH_PATH" >> tmp_header_file - fi - done - done - - # sort the required headers, eliminate copies - REQUIRED_HEADERS=$(sort -u tmp_header_file) - -done - -# remove the file that stores all required headers line by line -rm -f tmp_header_file - -# provide some output and copy everything to the new module -echo "" -echo -n "Number of required headers: " -echo "$REQUIRED_HEADERS" | wc -w -for HEADER in $REQUIRED_HEADERS; do - echo $HEADER - rsync -R $HEADER ../$MODULE_NAME -done -for DIR_PATH in $ALL_DIRECTORIES; do - rsync -rR $DIR_PATH ../$MODULE_NAME -done - -# delete all architecture-dependent files -cd ../$MODULE_NAME -find . -name Makefile.in -delete -find . -name Makefile -delete -find . -name '*.o' -delete -find . -wholename '*.deps/*' -delete -find . -path '*.deps' -delete -echo "Removed architecture-dependent files." - -# remove directories that are not required -sed -i '/(dune)/d' CMakeLists.txt -sed -i '/(src)/d' CMakeLists.txt -rm -rf dune/ -rm -rf src/ - -# create a list of the subfolders -EMPTY_DIR_NAME="." -rm -f tmp_subdir_file -for HEADER in $REQUIRED_HEADERS; do - - # move through every header, cutting off the last folder - while test "$HEADER" != $EMPTY_DIR_NAME; do - if test "$(dirname $HEADER)" != $EMPTY_DIR_NAME; then - dirname $HEADER >> tmp_subdir_file - fi - - HEADER=$(dirname $HEADER) - done - - # finally add the topmost folder - if test "$(basename $HEADER)" != $EMPTY_DIR_NAME; then - basename $HEADER >> tmp_subdir_file - fi -done -SUBDIR_LIST=$(sort -u tmp_subdir_file) -rm -f tmp_subdir_file - -# treat every subfolder -START_DIR=$PWD -INITIAL_SUBDIR_LIST="" -for SUBDIR in $SUBDIR_LIST; do - if test "$SUBDIR" = "$(basename $SUBDIR)"; then - INITIAL_SUBDIR_LIST="$INITIAL_SUBDIR_LIST $SUBDIR" - fi - - cd $SUBDIR - - # create lists of files and folders - FILE_LIST="" - DIR_LIST="" - for FILE_OR_FOLDER in $( ls ); do - if ([ -f $FILE_OR_FOLDER ]); then - FILE_LIST="$FILE_LIST $FILE_OR_FOLDER" - fi - - if ([ -d $FILE_OR_FOLDER ]); then - DIR_LIST="$DIR_LIST $FILE_OR_FOLDER" - fi - done - - # create CMakeLists.txt if not present - if ([ ! -f CMakeLists.txt ]); then - # add subfolder information - for DIR in $DIR_LIST; do - echo "add_subdirectory($DIR)" >> CMakeLists.txt - done - - # determine whether to add file information - ADD_FILE_INFO="0" - for FILE in $FILE_LIST; do - if ([ -f $FILE ]); then - ADD_FILE_INFO="1" - fi - done - - # add file information - if test "$ADD_FILE_INFO" = "1"; then - echo "" >> CMakeLists.txt - echo "install(FILES" >> CMakeLists.txt - - for FILE in $FILE_LIST; do - echo " $FILE" >> CMakeLists.txt - done - - echo " DESTINATION \${CMAKE_INSTALL_INCLUDEDIR}/$SUBDIR)" >> CMakeLists.txt - fi - fi - - cd $START_DIR - -done - -# update top CMakeLists.txt -for INITIAL_SUBDIR in $INITIAL_SUBDIR_LIST; do - sed -i '/add_subdirectory(doc)/a add_subdirectory('$INITIAL_SUBDIR')' CMakeLists.txt -done -echo "Updated and created CMakeLists.txt's." -cd .. - -# add includes to the automatically generated file *Macros.cmake -MACROS_FILE="$(find $MODULE_NAME -name '*Macros.cmake')" -if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_csv_file_links {} \; 2>/dev/null)" != ""; then - cp dumux-devel/cmake/modules/AddCSVFileLinks.cmake $MODULE_NAME/cmake/modules/. - echo "include(AddCSVFileLinks)" >>$MACROS_FILE -fi -if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_executable_all {} \; 2>/dev/null)" != ""; then - cp dumux-devel/cmake/modules/AddExecutableAll.cmake $MODULE_NAME/cmake/modules/. - echo "include(AddExecutableAll)" >>$MACROS_FILE -fi -if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_file_link {} \; 2>/dev/null)" != ""; then - cp dumux-devel/cmake/modules/AddFileLink.cmake $MODULE_NAME/cmake/modules/. - echo "include(AddFileLink)" >>$MACROS_FILE -fi -if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_folder_link {} \; 2>/dev/null)" != ""; then - cp dumux-devel/cmake/modules/AddFolderLink.cmake $MODULE_NAME/cmake/modules/. - echo "include(AddFolderLink)" >>$MACROS_FILE -fi -if test "$(find $MODULE_NAME -type f -name 'CMakeLists.txt' -exec grep add_gnuplot_file_links {} \; 2>/dev/null)" != ""; then - cp dumux-devel/cmake/modules/AddGnuplotFileLinks.cmake $MODULE_NAME/cmake/modules/. - echo "include(AddGnuplotFileLinks)" >>$MACROS_FILE -fi - -# create $README_FILE -README_FILE="README.md" -mv $MODULE_NAME/README $MODULE_NAME/$README_FILE -echo "This file has been created automatically. Please adapt it to your needs." >$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE -echo "===============================" >>$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE -echo "## Content" >>$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE -echo "The content of this DUNE module was extracted from the module \`$MODULE_DIR\`." >>$MODULE_NAME/$README_FILE -echo "In particular, the following subfolders of \`$MODULE_DIR\` have been extracted:" >>$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE -for DIR_PATH in $ALL_DIRECTORIES; do - echo "* \`$DIR_PATH\`," >>$MODULE_NAME/$README_FILE -done -echo "" >>$MODULE_NAME/$README_FILE -echo "Additionally, all headers in \`$MODULE_DIR\` that are required to build the" >>$MODULE_NAME/$README_FILE -echo "executables from the sources" >>$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE -for SOURCE in $ALL_SOURCES; do - echo "* \`$SOURCE\`," >>$MODULE_NAME/$README_FILE -done -echo "" >>$MODULE_NAME/$README_FILE -echo "have been extracted. You can configure the module just like any other DUNE" >>$MODULE_NAME/$README_FILE -echo "module by using \`dunecontrol\`. For building and running the executables," >>$MODULE_NAME/$README_FILE -echo "please go to the build folders corresponding to the sources listed above." >>$MODULE_NAME/$README_FILE -echo "" >>$MODULE_NAME/$README_FILE - -# get versions from modules we are depeding on -# create patches for un-pushed commits and uncomitted changes - -# try to find the dunecontrol script -if hash dunecontrol 2>/dev/null; then - DUNE_CONTROL="dunecontrol" -else - DUNE_CONTROL=$(find . -type f -executable -name 'dunecontrol') -fi -if test "$DUNE_CONTROL" = ""; then - echo "ERROR: Could not find dunecontrol." - echo "Be sure to either have dunecontrol in your search path" - echo "or to run this script from a directory that contains dunecontrol." - exit -fi -# get names of all modules we are depeding on -echo "Extracting dependencies to other Dune modules" -DEPENDING_MODULE_NAMES=$($DUNE_CONTROL --module=$MODULE_NAME 2>/dev/null | head -n 1) -DEPENDING_MODULE_NAMES=$(sed 's/--- going to build //' <<< "$DEPENDING_MODULE_NAMES") -DEPENDING_MODULE_NAMES=$(sed 's/ ---//' <<< "$DEPENDING_MODULE_NAMES") -# load script -CALL_FROM_EXTERNAL_SCRIPT="yes" -SCRIPT_DIR="${BASH_SOURCE%/*}" -if [[ ! -d "$SCRIPT_DIR" ]]; then - SCRIPT_DIR="$PWD"; -fi - -# output guidence for users -echo "" -echo "*************************************************************************" -echo "The extracted module is contained in the subfolder \"$MODULE_NAME\"." -echo "You can build it using \"dunecontrol ... --only=$MODULE_NAME all\"." -echo "*************************************************************************" -echo "BEFORE building, you can add the module to dumux-pub by something like:" -echo "(Rename module name if it does not match the AuthorLastNameYearx scheme" -echo "and commit it to the Git repository dumux-pub)" -echo "git clone https://git.iws.uni-stuttgart.de/dumux-pub/AuthorLastNameYearx.git" -echo "sed -i '/Module:/c\Module: AuthorLastNameYearx' $MODULE_NAME/dune.module" -echo "mv $MODULE_NAME/* dumux-pub/AuthorLastNameYearx/." -echo "cd AuthorLastNameYearx" -echo "git commit -a" -echo "git push" - -exit 0 -- GitLab From dcf7e667555d444b3b96ced3216914cabfd58caf Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Sun, 27 Jun 2021 23:59:15 +0200 Subject: [PATCH 15/47] [makeinstall] Add possibility Supresshints --- bin/extractmodule/makeinstallscript.py | 56 +++++++++++++------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index b71c55726a..2f7c2dd684 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -29,7 +29,8 @@ def makeInstallScript(path, ignoreUntracked=False, topFolderName='DUMUX', optsFile=None, - skipFolders=None): + skipFolders=None, + suppressHints=False): cwd = os.getcwd() modPath = os.path.abspath(os.path.join(cwd, path)) @@ -49,7 +50,6 @@ def makeInstallScript(path, print("\n-- Creating install script '{}' for module '{}' in folder '{}'" .format(instFileName, modName, modPath)) - print("\n-- Determining the dependencies") deps = getDependencies(modPath) if len(deps) > 0: @@ -77,7 +77,7 @@ def makeInstallScript(path, print('\nCaught exception: ' + str(err)) if 'untracked' in str(err): print('If you are certain that the untracked files are not needed for ' - 'the module installation, run this script with the -i flag.') + 'the module installation, run this script with the -i flag.') sys.exit(1) if len(versions) != len(depNames): @@ -87,15 +87,15 @@ def makeInstallScript(path, print(" on top of which we will generate the required patches") printVersionTable(versions) - print("\n-- Creating patches for unpublished commits and uncommitted changes") patches = getPatches(versions) # create patch files if len(patches) > 0: - print("-> Placing patches in the folder 'patches' in your module. You " - "should commit them to your repository in order for the install" - "script to work on other machines.") + if not suppressHints: + print("-> Placing patches in the folder 'patches' in your module. You " + "should commit them to your repository in order for the install" + "script to work on other machines.") patchesPath = os.path.join(modPath, 'patches') os.makedirs(patchesPath, exist_ok=True) @@ -245,11 +245,6 @@ def makeInstallScript(path, .format(depModName) + '\n') installFile.write('cd ..\n\n') - writeCommandWithErrorCheck( - 'mv ../{} .'.format(modFolder), - '-- Error: failed to move the new module to {}'.format(topFolderName) - ) - # write the module clone first in order for the patches to be present if modName in depNames: writeCloneModule(modName, modFolder) @@ -286,21 +281,22 @@ def makeInstallScript(path, subprocess.call(['chmod', 'u+x', instFileName]) # make script executable if len(patches) > 0: - print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" - " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" - " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" - " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") - if not skipFolders: + if not suppressHints: + print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" + " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" + " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" + " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") + + if not suppressHints: print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" - f" ## Installation\n" - f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" - f" Using `wget`, you can simply install all dependent modules by typing:\n" - f"\n" - f" ```sh\n" - f" wget {versions[modFolder]['remote']}/{instFileName}\n" - f" chmod u+x {instFileName}\n" - f" ./{instFileName}\n" - f" ```\n") + f" ## Installation\n" + f" The easiest way of installation is to use the script `{instFileName}` provided in this repository.\n" + f" Using `wget`, you can simply install all dependent modules by typing:\n" + f"\n" + f" ```sh\n" + f" wget {versions[modFolder]['remote']}/{instFileName}\n" + f" bash {instFileName}\n" + f" ```\n") if topFolderName: print(f" This will create a sub-folder `{topFolderName}`, clone all modules into it, configure the entire project and build the applications contained in this module.") @@ -342,6 +338,11 @@ if __name__ == '__main__': parser.add_argument('-s', '--skipfolders', required=False, default=None, help='a list of module folders to be skipped') + + parser.add_argument('-d', '--suppresshints', + required=False, default=False, + help='if output needs to be suppressed') + cmdArgs = vars(parser.parse_args()) makeInstallScript( @@ -350,5 +351,6 @@ if __name__ == '__main__': ignoreUntracked=cmdArgs.get('ignoreuntracked', False), topFolderName=cmdArgs.get('topfoldername', None), optsFile=cmdArgs.get('optsFile', None), - skipFolders=cmdArgs.get('skipfolders', None) + skipFolders=cmdArgs.get('skipfolders', None), + suppressHints=cmdArgs.get('suppresshints', False), ) -- GitLab From bd9e743e15911317d21019af0054db0449ccbb00 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Thu, 24 Jun 2021 12:17:44 +0200 Subject: [PATCH 16/47] [bin][extractmodule] Add reminder to replace placeholder in README --- bin/extractmodule/extract_as_new_module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 6ca8ecdea0..013bd74b1d 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -480,3 +480,5 @@ if __name__ == "__main__": run_from_mod('git init') run_from_mod('git add .') run_from_mod('git commit -m "Initial commit"') + print("Please remember to replace the placeholder $remoteurl$ in README installation\n" + "with the correct URL after your remote git repository is created.") -- GitLab From cefcaa99af9101a1f58588cc5e4217e331072afa Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Fri, 25 Jun 2021 23:00:11 +0200 Subject: [PATCH 17/47] [bin][makeinstall] Pass patch information directly as string into install script --- bin/extractmodule/makeinstallscript.py | 82 ++++++++------------------ 1 file changed, 23 insertions(+), 59 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 2f7c2dd684..4fa8ed867c 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -90,44 +90,6 @@ def makeInstallScript(path, print("\n-- Creating patches for unpublished commits and uncommitted changes") patches = getPatches(versions) - # create patch files - if len(patches) > 0: - if not suppressHints: - print("-> Placing patches in the folder 'patches' in your module. You " - "should commit them to your repository in order for the install" - "script to work on other machines.") - patchesPath = os.path.join(modPath, 'patches') - os.makedirs(patchesPath, exist_ok=True) - - def getPatchFileNameAndRelPath(depModPath, targetName): - i = 1 - fileName = os.path.join(patchesPath, targetName + '.patch') - while os.path.exists(fileName): - fileName = os.path.join(patchesPath, - targetName + '_' + str(i) + '.patch') - i += 1 - - return fileName, os.path.relpath(fileName, depModPath) - - def writePatch(depModPath, depModName, type): - patchName = depModName + '_' + type - pPath, pRelPath = getPatchFileNameAndRelPath(depModPath, patchName) - patches[depModPath][type + '_relpath'] = pRelPath - open(pPath, 'w').write(patches[depModPath][type]) - return pPath - - print("-> Created patch files:") - for depModPath in patches.keys(): - depModName = getModuleInfo(depModPath, "Module") - if 'unpublished' in patches[depModPath]: - print(' '*3 + writePatch(depModPath, depModName, 'unpublished')) - if 'uncommitted' in patches[depModPath]: - print(' '*3 + writePatch(depModPath, depModName, 'uncommitted')) - - else: - print("-> No Patches required") - - # write installation shell script (switch to relative paths) versions = {os.path.relpath(p, modParentPath): v for p, v in versions.items()} patches = {os.path.relpath(p, modParentPath): v for p, v in patches.items()} @@ -156,6 +118,18 @@ def makeInstallScript(path, '}\n\n' ) + # write patch information into install script + if len(patches) > 0: + for depModPath in patches.keys(): + if 'unpublished' in patches[depModPath]: + installFile.write("cat >> unpublished.patch <<'EOF'\n" + + patches[depModPath]['unpublished'] + + "\nEOF\n") + if 'uncommitted' in patches[depModPath]: + installFile.write("cat >> uncommitted.patch <<'EOF'\n" + + patches[depModPath]['uncommitted'] + + "\nEOF\n") + unitIndentation = ' '*4 def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): @@ -220,27 +194,6 @@ def makeInstallScript(path, .format(versions[depModFolder]['revision'], depModName) ) - # write section on application of patches - def writeApplyPatch(patchRelPath): - patchError = '--Error: patch {} was not found'.format(patchRelPath) - installFile.write('if [ -f {} ]; then\n'.format(patchRelPath)) - writeCommandWithErrorCheck( - 'git apply {}'.format(patchRelPath), - '--Error: failed to apply patch {} in module {}' - .format(patchRelPath, depModName), - indentationLevel=1 - ) - installFile.write("else\n" - f"{unitIndentation}{exitFunc} " - f"\"{patchError}\".\n") - installFile.write('fi\n') - - if depModFolder in patches: - if'unpublished_relpath' in patches[depModFolder]: - writeApplyPatch(patches[depModFolder]['unpublished_relpath']) - if 'uncommitted_relpath' in patches[depModFolder]: - writeApplyPatch(patches[depModFolder]['uncommitted_relpath']) - installFile.write('echo "-- Successfully set up the module {} \\n"\n' .format(depModName) + '\n') installFile.write('cd ..\n\n') @@ -252,6 +205,17 @@ def makeInstallScript(path, if depModName != modName: writeCloneModule(depModName, depModFolder) + for patchtype in ['unpublished', 'uncommited']: + installFile.write('if [ -f {}.patch ]; then\n'.format(patchtype)) + writeCommandWithErrorCheck( + 'git apply {}.patch'.format(patchtype), + '--Error: failed to apply patch {}.patch'.format(patchtype), + indentationLevel=1) + installFile.write("else\n" + f"{unitIndentation} " + + 'echo "No patch {}.patch is found".\n'.format(patchtype)) + installFile.write('fi\n') + # write configure command installFile.write('echo "-- All modules haven been cloned successfully. ' 'Configuring project..."\n') -- GitLab From 2bf4baa0ce111d8972c9db6552c651279888fb3e Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Sun, 27 Jun 2021 04:29:00 +0200 Subject: [PATCH 18/47] [bin][makeinstall] Write the installation process in a function with args url sha and patch --- bin/extractmodule/makeinstallscript.py | 118 +++++++++++++------------ 1 file changed, 63 insertions(+), 55 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 4fa8ed867c..a33357f6c5 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -107,28 +107,82 @@ def makeInstallScript(path, " together with all dependencies.\n" "\n".format(modName)) - exitFunc = 'exitWith' + # write function to install the new module and its dependencies into install script installFile.write( - '# defines a function to exit with error message\n' - '{} ()\n'.format(exitFunc) - + + '# defines the function to install module with urls, shas and patches\n' + 'InstallModule()\n' '{\n' - ' echo "\\n$1"\n' - ' exit 1\n' + ' URLS=$1\n' + ' DEPFOLDERS=$2\n' + ' DEPBRANCHES=$3\n' + ' DEPSHAS=$4\n' + ' PATCHES=$5\n\n' + ' for url in ${URLS[@]}; do\n' + ' if ! git clone $url; then\n' + ' echo "--Error: failed to clone $url"\n' + ' fi\n' + ' done\n\n' + ' for index in ${!DEPFOLDERS[@]}; do\n' + ' if ! cd ${DEPFOLDERS[index]}; then\n' + ' echo "Error: could not enter folder ${DEPFOLDERS[index]}"\n' + ' fi\n' + ' if ! git checkout ${DEPBRANCHES[index]}; then\n' + ' echo "-- Error: failed to check out branch ${DEPBRANCHES[index]} in ${DEPFOLDERS[index]}."\n' + ' fi\n' + ' if ! git reset --hard ${DEPSHAS[index]}; then\n' + ' echo "-- Error: failed to check out commit ${DEPSHAS[index]} in module ${DEPFOLDERS[index]}."\n' + ' fi\n' + ' cd ..\n' + ' done\n\n' + ' for patch in ${PATCHES[@]}; do\n' + ' if ! [ -f $patch ]; then\n' + f' if ! git apply {"../" if topFolderName else ""}$patch; then\n' + ' echo "--Error: failed to apply patch $patch"\n' + ' fi\n' + ' else\n' + ' echo "--Error no patch $patch is found"\n' + ' fi\n' + ' done\n' '}\n\n' ) + # write urls, depfolders/branches/shas and patches into install script + installFile.write('# Information about url, folder/branch/sha and patch\n') + folders = depFolders + if modName in depNames: + folders.append(modName) + installFile.write('URLS=(\n') + for dep in folders: + installFile.write(versions[dep]['remote'] + '\n') + installFile.write(')\n\n') + + installFile.write('DEPFOLDERS=(\n') + installFile.write('\n'.join(folders)) + installFile.write('\n)\n\n') + + installFile.write('DEPBRANCHES=(\n') + for dep in folders: + installFile.write(versions[dep]['branch'] + '\n') + installFile.write(')\n\n') + + installFile.write('DEPSHAS=(\n') + for dep in folders: + installFile.write(versions[dep]['revision'] + '\n') + installFile.write(')\n\n') + + installFile.write('PATCHES=(unpublished.patch uncommitted.patch)\n\n') + # write patch information into install script if len(patches) > 0: for depModPath in patches.keys(): if 'unpublished' in patches[depModPath]: installFile.write("cat >> unpublished.patch <<'EOF'\n" + patches[depModPath]['unpublished'] - + "\nEOF\n") + + "EOF\n") if 'uncommitted' in patches[depModPath]: installFile.write("cat >> uncommitted.patch <<'EOF'\n" + patches[depModPath]['uncommitted'] - + "\nEOF\n") + + "EOF\n") unitIndentation = ' '*4 @@ -168,53 +222,7 @@ def makeInstallScript(path, ' script is executed.\n\n' ) - def writeCloneModule(depModName, depModFolder): - installFile.write('# ' + depModName + '\n') - installFile.write('# ' + versions[depModFolder]['branch'] + ' # ' - + versions[depModFolder]['revision'] + ' # ' - + versions[depModFolder]['date'] + ' # ' - + versions[depModFolder]['author'] + '\n') - - writeCommandWithErrorCheck( - 'git clone {}'.format(versions[depModFolder]['remote']), - '-- Error: failed to clone {}.'.format(depModName) - ) - writeCommandWithErrorCheck( - 'cd {}'.format(depModFolder), - '-- Error: could not enter folder {}.'.format(depModFolder) - ) - writeCommandWithErrorCheck( - 'git checkout {}'.format(versions[depModFolder]['branch']), - '-- Error: failed to check out branch {} in module {}.' - .format(versions[depModFolder]['branch'], depModName) - ) - writeCommandWithErrorCheck( - 'git reset --hard {}'.format(versions[depModFolder]['revision']), - '-- Error: failed to check out commit {} in module {}.' - .format(versions[depModFolder]['revision'], depModName) - ) - - installFile.write('echo "-- Successfully set up the module {} \\n"\n' - .format(depModName) + '\n') - installFile.write('cd ..\n\n') - - # write the module clone first in order for the patches to be present - if modName in depNames: - writeCloneModule(modName, modFolder) - for depModName, depModFolder in zip(depNames, depFolders): - if depModName != modName: - writeCloneModule(depModName, depModFolder) - - for patchtype in ['unpublished', 'uncommited']: - installFile.write('if [ -f {}.patch ]; then\n'.format(patchtype)) - writeCommandWithErrorCheck( - 'git apply {}.patch'.format(patchtype), - '--Error: failed to apply patch {}.patch'.format(patchtype), - indentationLevel=1) - installFile.write("else\n" - f"{unitIndentation} " - + 'echo "No patch {}.patch is found".\n'.format(patchtype)) - installFile.write('fi\n') + installFile.write('InstallModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS $PATCHES\n\n') # write configure command installFile.write('echo "-- All modules haven been cloned successfully. ' -- GitLab From 2ab3685c748a04ddbf4f0172787331eee17c57b6 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Sun, 27 Jun 2021 15:31:27 +0200 Subject: [PATCH 19/47] [bin][makeinstall] Fix patch paths --- bin/extractmodule/makeinstallscript.py | 97 ++++++++++++++++---------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index a33357f6c5..73a41392a8 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -112,44 +112,49 @@ def makeInstallScript(path, '# defines the function to install module with urls, shas and patches\n' 'InstallModule()\n' '{\n' - ' URLS=$1\n' - ' DEPFOLDERS=$2\n' - ' DEPBRANCHES=$3\n' - ' DEPSHAS=$4\n' - ' PATCHES=$5\n\n' - ' for url in ${URLS[@]}; do\n' - ' if ! git clone $url; then\n' - ' echo "--Error: failed to clone $url"\n' - ' fi\n' - ' done\n\n' - ' for index in ${!DEPFOLDERS[@]}; do\n' - ' if ! cd ${DEPFOLDERS[index]}; then\n' - ' echo "Error: could not enter folder ${DEPFOLDERS[index]}"\n' - ' fi\n' - ' if ! git checkout ${DEPBRANCHES[index]}; then\n' - ' echo "-- Error: failed to check out branch ${DEPBRANCHES[index]} in ${DEPFOLDERS[index]}."\n' - ' fi\n' - ' if ! git reset --hard ${DEPSHAS[index]}; then\n' - ' echo "-- Error: failed to check out commit ${DEPSHAS[index]} in module ${DEPFOLDERS[index]}."\n' - ' fi\n' - ' cd ..\n' - ' done\n\n' - ' for patch in ${PATCHES[@]}; do\n' - ' if ! [ -f $patch ]; then\n' - f' if ! git apply {"../" if topFolderName else ""}$patch; then\n' - ' echo "--Error: failed to apply patch $patch"\n' - ' fi\n' - ' else\n' - ' echo "--Error no patch $patch is found"\n' - ' fi\n' - ' done\n' + ' URLS=$1\n' + ' DEPFOLDERS=$2\n' + ' DEPBRANCHES=$3\n' + ' DEPSHAS=$4\n' + ' PATCHES=$5\n' + ' PATCHFOLDERS=$6\n\n' + ' for url in ${URLS[@]}; do\n' + ' if ! git clone $url; then\n' + ' echo "--Error: failed to clone $url"\n' + ' fi\n' + ' done\n\n' + ' for index in ${!DEPFOLDERS[@]}; do\n' + ' if ! cd ${DEPFOLDERS[index]}; then\n' + ' echo "Error: could not enter folder ${DEPFOLDERS[index]}"\n' + ' fi\n' + ' if ! git checkout ${DEPBRANCHES[index]}; then\n' + ' echo "-- Error: failed to check out branch ${DEPBRANCHES[index]} in ${DEPFOLDERS[index]}."\n' + ' fi\n' + ' if ! git reset --hard ${DEPSHAS[index]}; then\n' + ' echo "-- Error: failed to check out commit ${DEPSHAS[index]} in module ${DEPFOLDERS[index]}."\n' + ' fi\n' + ' cd ..\n' + ' done\n\n' + ' for i in ${!PATCHES[@]}; do\n' + # # associative array is for bash 4 or newer + # ' if ! [ -f ${PATCHES[$module]} ]; then\n' + # ' cd $module\n' + # f' if ! git apply {"../" if topFolderName else ""}$' + # '{PATCHES[$module]}; then\n' + ' cd ${PATCHFOLDERS[i]}\n' + f' if ! git apply {"../" if topFolderName else ""}$' + '{PATCHES[i]}; then\n' + ' echo "--Error: failed to apply patch $patch"\n' + ' fi\n' + ' cd ..\n' + ' done\n' '}\n\n' ) # write urls, depfolders/branches/shas and patches into install script installFile.write('# Information about url, folder/branch/sha and patch\n') folders = depFolders - if modName in depNames: + if modName in depNames and modName not in folders: folders.append(modName) installFile.write('URLS=(\n') for dep in folders: @@ -170,19 +175,37 @@ def makeInstallScript(path, installFile.write(versions[dep]['revision'] + '\n') installFile.write(')\n\n') - installFile.write('PATCHES=(unpublished.patch uncommitted.patch)\n\n') - # write patch information into install script if len(patches) > 0: + patchRelPath = [] + patchModule = [] for depModPath in patches.keys(): + depModName = getModuleInfo(depModPath, "Module") if 'unpublished' in patches[depModPath]: - installFile.write("cat >> unpublished.patch <<'EOF'\n" + installFile.write("cat >> {}_unpublished.patch <<'EOF'\n".format(depModName) + patches[depModPath]['unpublished'] + "EOF\n") + patchModule.append(depModPath) + patchRelPath.append(os.path.relpath("{}_unpublished.patch".format(depModName), depModPath)) if 'uncommitted' in patches[depModPath]: - installFile.write("cat >> uncommitted.patch <<'EOF'\n" + installFile.write("cat >> {}_uncommitted.patch <<'EOF'\n".format(depModName) + patches[depModPath]['uncommitted'] + "EOF\n") + patchRelPath.append(os.path.relpath("{}_uncommitted.patch".format(depModName), depModPath)) + patchModule.append(depModPath) + + # associative Arrays requires bash 4 or newer! + # installFile.write('declare -A PATCHES\n') + # for module, path in zip(patchModule, patchRelPath): + # installFile.write('PATCHES[' + module + ']=' + path + '\n') + + installFile.write('PATCHFOLDERS=(\n') + installFile.write('\n'.join(patchModule)) + installFile.write('\n)\n\n') + + installFile.write('PATCHES=(\n') + installFile.write('\n'.join(patchRelPath)) + installFile.write('\n)\n\n') unitIndentation = ' '*4 @@ -222,7 +245,7 @@ def makeInstallScript(path, ' script is executed.\n\n' ) - installFile.write('InstallModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS $PATCHES\n\n') + installFile.write('InstallModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS $PATCHES $PATCHFOLDERS\n\n') # write configure command installFile.write('echo "-- All modules haven been cloned successfully. ' -- GitLab From b512cd21ff17655cb41300a8f47963734ad0f342 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Mon, 28 Jun 2021 00:12:07 +0200 Subject: [PATCH 20/47] [bin][makeinstall] Remove bash4 comments and redundant comma --- bin/extractmodule/makeinstallscript.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 73a41392a8..674dc8b429 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -136,11 +136,6 @@ def makeInstallScript(path, ' cd ..\n' ' done\n\n' ' for i in ${!PATCHES[@]}; do\n' - # # associative array is for bash 4 or newer - # ' if ! [ -f ${PATCHES[$module]} ]; then\n' - # ' cd $module\n' - # f' if ! git apply {"../" if topFolderName else ""}$' - # '{PATCHES[$module]}; then\n' ' cd ${PATCHFOLDERS[i]}\n' f' if ! git apply {"../" if topFolderName else ""}$' '{PATCHES[i]}; then\n' @@ -194,11 +189,6 @@ def makeInstallScript(path, patchRelPath.append(os.path.relpath("{}_uncommitted.patch".format(depModName), depModPath)) patchModule.append(depModPath) - # associative Arrays requires bash 4 or newer! - # installFile.write('declare -A PATCHES\n') - # for module, path in zip(patchModule, patchRelPath): - # installFile.write('PATCHES[' + module + ']=' + path + '\n') - installFile.write('PATCHFOLDERS=(\n') installFile.write('\n'.join(patchModule)) installFile.write('\n)\n\n') @@ -347,5 +337,5 @@ if __name__ == '__main__': topFolderName=cmdArgs.get('topfoldername', None), optsFile=cmdArgs.get('optsFile', None), skipFolders=cmdArgs.get('skipfolders', None), - suppressHints=cmdArgs.get('suppresshints', False), + suppressHints=cmdArgs.get('suppresshints', False) ) -- GitLab From 47e8bbf787110ae716c848f387b437343f453d16 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 27 Jun 2021 22:38:01 +0000 Subject: [PATCH 21/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/makeinstallscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 674dc8b429..e222a46722 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -110,7 +110,7 @@ def makeInstallScript(path, # write function to install the new module and its dependencies into install script installFile.write( '# defines the function to install module with urls, shas and patches\n' - 'InstallModule()\n' + 'installModule()\n' '{\n' ' URLS=$1\n' ' DEPFOLDERS=$2\n' -- GitLab From 0540045a5972d8da4f66fb82cfac2b9a57d763c0 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Mon, 28 Jun 2021 00:48:14 +0200 Subject: [PATCH 22/47] [bin][extract] Delete the generation of stand-alone install script --- bin/extractmodule/extract_as_new_module.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 013bd74b1d..25bfc53c99 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -467,10 +467,6 @@ if __name__ == "__main__": run_from_mod('git remote add origin {}'.format(remoteurl)) run_from_mod('git push -u origin master') - if query_yes_no("Do you also want to create an one-click install script?"): - makeInstallScript(new_module_path, ignoreUntracked=True, suppressHints=True) - print("The one-click install script {} is created in your current folder.".format(install_script_name)) - # output guidance for users to create install script manually else: remoteurl = "{$remoteurl$} ({$remoteurl$} is the URL for your remote git repository)" -- GitLab From 0281b18edbb0612b1a3c62af681e2da32996e725 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Tue, 29 Jun 2021 09:43:25 +0200 Subject: [PATCH 23/47] [extractmodule] Adapt the installation instructions to match the current script --- bin/extractmodule/extract_as_new_module.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 25bfc53c99..590c669bd0 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -186,16 +186,20 @@ def info_readme_installation(remoteurl, install_script_name, new_module_name): ## Installation -The easiest way of installation is to use the install script `{install_script_name}` -provided in this repository. -You can simply clone the remote and install all dependent modules by typing: +The installation procedure is done as follows : +Create a root folder, e.g. `$NewFolder$`, enter the previously created folder +clone the remote and use the install script `{install_script_name}` +provided in this repository to install all dependent modules. ```sh +mkdir $NewFolder$ +cd $NewFolder$ git clone {remoteurl} ./{new_module_name}/{install_script_name} ``` -This will create a sub-folder `DUMUX`, clone all modules into it. +This will clone all modules into your created folder `$NewFolder$`, +configure your module with `dunecontrol` and build tests. """ -- GitLab From a13df8e51f6d027f1842cb5b0b0ec95372918ca9 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Tue, 29 Jun 2021 09:50:20 +0200 Subject: [PATCH 24/47] [bin][util] Add function to write the content into the generated shell install script --- bin/extractmodule/util.py | 181 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index 6e45b9a50f..f2d9beda23 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -8,6 +8,7 @@ try: sys.path.append(os.path.join(path, '../util')) from common import callFromPath, runCommand + from getmoduleinfo import getModuleInfo except Exception: sys.exit('Could not import common module') @@ -135,3 +136,183 @@ def versionTable(versions): def printVersionTable(versions): print(versionTable(versions=versions)) + + +def writeShellInstallScript(instFileName, + modName, modFolder, + folders, versions, + patches, patchModule, patchRelPath, + topFolderName, optsRelPath="dumux/cmake.opts"): + """ + function to write the content into the generated shell install script + + Keyword Arguments: + + - instFileName -- The name of the generated shell install script + + - modName -- The name of the module to be installed + - modFolder -- The folder containing the module to be installed + + - folders -- The list containing the folders of the module to be + installed and all its dependencies + - versions -- The persistent, remotely available git versions for + the module to be installed and all its dependencies + + - patches -- The patches for unpublished commits and uncommitted changes + - patchModule -- The paths for the modules which has unpublished commits and uncommited changes + - patchRelPath -- The realative paths of the generated patch files + + - topFolderName -- The of the folder that the install script creates upon execution to install the module in. + If an empty string is passed, no folder will be created and installation happens in place. + - optsRelPath -- The relative path of custom opts file called by 'dunecontrol' + + """ + + with open(instFileName, 'w') as installFile: + installFile.write("#!/bin/bash\n\n") + installFile.write("#"*80 + "\n") + installFile.write("# This script installs the module '{}'" + " together with all dependencies.\n" + "\n".format(modName)) + + # write function to install the new module and its dependencies into install script + installFile.write( + '# defines the function to install module with urls, shas and patches\n' + 'installModule()\n' + '{\n' + ' URLS=$1\n' + ' DEPFOLDERS=$2\n' + ' DEPBRANCHES=$3\n' + ' DEPSHAS=$4\n' + ' PATCHES=$5\n' + ' PATCHFOLDERS=$6\n\n' + ' for url in ${URLS[@]}; do\n' + ' if ! git clone $url; then\n' + ' echo "--Error: failed to clone $url"\n' + ' fi\n' + ' done\n\n' + ' for index in ${!DEPFOLDERS[@]}; do\n' + ' if ! cd ${DEPFOLDERS[index]}; then\n' + ' echo "Error: could not enter folder ${DEPFOLDERS[index]}"\n' + ' fi\n' + ' if ! git checkout ${DEPBRANCHES[index]}; then\n' + ' echo "-- Error: failed to check out branch ${DEPBRANCHES[index]} in ${DEPFOLDERS[index]}."\n' + ' fi\n' + ' if ! git reset --hard ${DEPSHAS[index]}; then\n' + ' echo "-- Error: failed to check out commit ${DEPSHAS[index]} in module ${DEPFOLDERS[index]}."\n' + ' fi\n' + ' cd ..\n' + ' done\n\n' + ' for i in ${!PATCHES[@]}; do\n' + ' cd ${PATCHFOLDERS[i]}\n' + f' if ! git apply {"../" if topFolderName else ""}$' + '{PATCHES[i]}; then\n' + ' echo "--Error: failed to apply patch $patch"\n' + ' fi\n' + ' cd ..\n' + ' done\n' + '}\n\n' + ) + + # write urls, depfolders/branches/shas and patches into install script + installFile.write('# Information about url, folder/branch/sha and patch\n') + installFile.write('URLS=(\n') + for dep in folders: + installFile.write(versions[dep]['remote'] + '\n') + installFile.write(')\n\n') + + installFile.write('DEPFOLDERS=(\n') + installFile.write('\n'.join(folders)) + installFile.write('\n)\n\n') + + installFile.write('DEPBRANCHES=(\n') + for dep in folders: + installFile.write(versions[dep]['branch'] + '\n') + installFile.write(')\n\n') + + installFile.write('DEPSHAS=(\n') + for dep in folders: + installFile.write(versions[dep]['revision'] + '\n') + installFile.write(')\n\n') + + # write patch information into install script + installFile.write('PATCHFOLDERS=(\n') + installFile.write('\n'.join(patchModule)) + installFile.write('\n)\n\n') + + installFile.write('PATCHES=(\n') + installFile.write('\n'.join(patchRelPath)) + installFile.write('\n)\n\n') + + for depModPath in set(patchModule): + depModName = getModuleInfo(depModPath, "Module") + if 'unpublished' in patches[depModPath]: + installFile.write("cat >> {}_unpublished.patch <<'EOF'\n".format(depModName) + + patches[depModPath]['unpublished'] + + "EOF\n") + if 'uncommitted' in patches[depModPath]: + installFile.write("cat >> {}_uncommitted.patch <<'EOF'\n".format(depModName) + + patches[depModPath]['uncommitted'] + + "EOF\n") + + def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): + unitIndentation = ' '*4 + indent = unitIndentation*indentationLevel + nextIndent = indent + unitIndentation + installFile.write( + f"{indent}if ! {command}; then\n" + f"{nextIndent}echo \"{errorMessage}\"\n" + "fi\n" + ) + + if topFolderName: + installFile.write( + '# Everything will be installed into a new folder "{}".\n' + .format(topFolderName) + ) + installFile.write( + 'echo "Creating the folder {} to install the modules in"\n' + .format(topFolderName) + ) + + writeCommandWithErrorCheck( + 'mkdir -p {}'.format(topFolderName), + '--Error: could not create top folder {}'.format(topFolderName) + ) + + writeCommandWithErrorCheck( + 'cd {}'.format(topFolderName), + '--Error: could not enter top folder {}'.format(topFolderName)) + + installFile.write('\n') + else: + installFile.write( + '# Everything will be installed inside the folder from which the' + ' script is executed.\n\n' + ) + + installFile.write('installModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS $PATCHES $PATCHFOLDERS\n\n') + + # write configure command + installFile.write('echo "-- All modules haven been cloned successfully. ' + 'Configuring project..."\n') + writeCommandWithErrorCheck( + './dune-common/bin/dunecontrol --opts=dumux/cmake.opts all'.format(optsRelPath), + '--Error: could not configure project' + ) + + # build the tests of the module + installFile.write( + '\n' + 'echo "-- Configuring successful. Compiling applications..."\n' + ) + writeCommandWithErrorCheck( + 'cd {}/build-cmake'.format(modFolder), + '--Error: could not enter build directory at {}/build-cmake' + .format(modFolder) + ) + writeCommandWithErrorCheck( + 'make build_tests', + '--Error: applications could not be compiled. ' + 'Please try to compile them manually.' + ) -- GitLab From 2d252918e287dd0cf2acc64cdf91adfaefdc68af Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Tue, 29 Jun 2021 09:51:37 +0200 Subject: [PATCH 25/47] [bin][makeinstall] Call function writeShellInstallScript from util --- bin/extractmodule/makeinstallscript.py | 183 +++---------------------- 1 file changed, 22 insertions(+), 161 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index e222a46722..9d71c62f89 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -11,6 +11,7 @@ import subprocess from util import getPersistentVersions from util import printVersionTable from util import getPatches +from util import writeShellInstallScript try: path = os.path.split(os.path.abspath(__file__))[0] @@ -99,167 +100,27 @@ def makeInstallScript(path, optsPath = os.path.abspath(os.path.join(os.getcwd(), optsFile)) optsRelPath = os.path.relpath(optsPath, modParentPath) - # TODO: add support for different install script languages (e.g. Python). - with open(instFileName, 'w') as installFile: - installFile.write("#!/bin/bash\n\n") - installFile.write("#"*80 + "\n") - installFile.write("# This script installs the module '{}'" - " together with all dependencies.\n" - "\n".format(modName)) - - # write function to install the new module and its dependencies into install script - installFile.write( - '# defines the function to install module with urls, shas and patches\n' - 'installModule()\n' - '{\n' - ' URLS=$1\n' - ' DEPFOLDERS=$2\n' - ' DEPBRANCHES=$3\n' - ' DEPSHAS=$4\n' - ' PATCHES=$5\n' - ' PATCHFOLDERS=$6\n\n' - ' for url in ${URLS[@]}; do\n' - ' if ! git clone $url; then\n' - ' echo "--Error: failed to clone $url"\n' - ' fi\n' - ' done\n\n' - ' for index in ${!DEPFOLDERS[@]}; do\n' - ' if ! cd ${DEPFOLDERS[index]}; then\n' - ' echo "Error: could not enter folder ${DEPFOLDERS[index]}"\n' - ' fi\n' - ' if ! git checkout ${DEPBRANCHES[index]}; then\n' - ' echo "-- Error: failed to check out branch ${DEPBRANCHES[index]} in ${DEPFOLDERS[index]}."\n' - ' fi\n' - ' if ! git reset --hard ${DEPSHAS[index]}; then\n' - ' echo "-- Error: failed to check out commit ${DEPSHAS[index]} in module ${DEPFOLDERS[index]}."\n' - ' fi\n' - ' cd ..\n' - ' done\n\n' - ' for i in ${!PATCHES[@]}; do\n' - ' cd ${PATCHFOLDERS[i]}\n' - f' if ! git apply {"../" if topFolderName else ""}$' - '{PATCHES[i]}; then\n' - ' echo "--Error: failed to apply patch $patch"\n' - ' fi\n' - ' cd ..\n' - ' done\n' - '}\n\n' - ) - - # write urls, depfolders/branches/shas and patches into install script - installFile.write('# Information about url, folder/branch/sha and patch\n') - folders = depFolders - if modName in depNames and modName not in folders: - folders.append(modName) - installFile.write('URLS=(\n') - for dep in folders: - installFile.write(versions[dep]['remote'] + '\n') - installFile.write(')\n\n') - - installFile.write('DEPFOLDERS=(\n') - installFile.write('\n'.join(folders)) - installFile.write('\n)\n\n') - - installFile.write('DEPBRANCHES=(\n') - for dep in folders: - installFile.write(versions[dep]['branch'] + '\n') - installFile.write(')\n\n') - - installFile.write('DEPSHAS=(\n') - for dep in folders: - installFile.write(versions[dep]['revision'] + '\n') - installFile.write(')\n\n') - - # write patch information into install script - if len(patches) > 0: - patchRelPath = [] - patchModule = [] - for depModPath in patches.keys(): - depModName = getModuleInfo(depModPath, "Module") - if 'unpublished' in patches[depModPath]: - installFile.write("cat >> {}_unpublished.patch <<'EOF'\n".format(depModName) - + patches[depModPath]['unpublished'] - + "EOF\n") - patchModule.append(depModPath) - patchRelPath.append(os.path.relpath("{}_unpublished.patch".format(depModName), depModPath)) - if 'uncommitted' in patches[depModPath]: - installFile.write("cat >> {}_uncommitted.patch <<'EOF'\n".format(depModName) - + patches[depModPath]['uncommitted'] - + "EOF\n") - patchRelPath.append(os.path.relpath("{}_uncommitted.patch".format(depModName), depModPath)) - patchModule.append(depModPath) - - installFile.write('PATCHFOLDERS=(\n') - installFile.write('\n'.join(patchModule)) - installFile.write('\n)\n\n') - - installFile.write('PATCHES=(\n') - installFile.write('\n'.join(patchRelPath)) - installFile.write('\n)\n\n') - - unitIndentation = ' '*4 - - def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): - indent = unitIndentation*indentationLevel - nextIndent = indent + unitIndentation - installFile.write( - f"{indent}if ! {command}; then\n" - f"{nextIndent}echo \"{errorMessage}\"\n" - "fi\n" - ) - - if topFolderName: - installFile.write( - '# Everything will be installed into a new folder "{}".\n' - .format(topFolderName) - ) - installFile.write( - 'echo "Creating the folder {} to install the modules in"\n' - .format(topFolderName) - - ) - - writeCommandWithErrorCheck( - 'mkdir -p {}'.format(topFolderName), - '--Error: could not create top folder {}'.format(topFolderName) - ) - - writeCommandWithErrorCheck( - 'cd {}'.format(topFolderName), - '--Error: could not enter top folder {}'.format(topFolderName)) - - installFile.write('\n') - else: - installFile.write( - '# Everything will be installed inside the folder from which the' - ' script is executed.\n\n' - ) - - installFile.write('InstallModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS $PATCHES $PATCHFOLDERS\n\n') - - # write configure command - installFile.write('echo "-- All modules haven been cloned successfully. ' - 'Configuring project..."\n') - writeCommandWithErrorCheck( - './dune-common/bin/dunecontrol --opts={} all'.format(optsRelPath), - '--Error: could not configure project' - ) - - # build the tests of the module - installFile.write( - '\n' - 'echo "-- Configuring successful. Compiling applications..."\n' - ) - writeCommandWithErrorCheck( - 'cd {}/build-cmake'.format(modFolder), - '--Error: could not enter build directory at {}/build-cmake' - .format(modFolder) - ) - writeCommandWithErrorCheck( - 'make build_tests', - '--Error: applications could not be compiled. ' - 'Please try to compile them manually.' - ) + folders = depFolders + if modName in depNames and modName not in folders: + folders.append(modName) + + # specific Module required patch and the patch path + patchRelPath = [] + patchModule = [] + for depModPath in patches.keys(): + depModName = getModuleInfo(depModPath, "Module") + if 'unpublished' in patches[depModPath]: + patchModule.append(depModPath) + patchRelPath.append(os.path.relpath("{}_unpublished.patch".format(depModName), depModPath)) + if 'uncommitted' in patches[depModPath]: + patchRelPath.append(os.path.relpath("{}_uncommitted.patch".format(depModName), depModPath)) + patchModule.append(depModPath) + + writeShellInstallScript(instFileName, + modName, modFolder, + folders, versions, + patches, patchModule, patchRelPath, + topFolderName, optsRelPath) print("\n-- Successfully created install script file " + instFileName) -- GitLab From 748e59ffc8d28e11009d100669094b0d21046581 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Tue, 29 Jun 2021 20:13:00 +0200 Subject: [PATCH 26/47] [bin][util] Add the function to generate install script for a module in python --- bin/extractmodule/util.py | 186 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index f2d9beda23..23e5facaa8 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -316,3 +316,189 @@ def writeShellInstallScript(instFileName, '--Error: applications could not be compiled. ' 'Please try to compile them manually.' ) + + +def writePythonInstallScript(instFileName, + modName, modFolder, + folders, versions, + patches, patchModule, patchRelPath, + topFolderName, optsRelPath="dumux/cmake.opts"): + """ + function to write the content into the generated python install script + + Keyword Arguments: + + - instFileName -- The name of the generated python install script + + - modName -- The name of the module to be installed + - modFolder -- The folder containing the module to be installed + + - folders -- The list containing the folders of the module to be + installed and all its dependencies + - versions -- The persistent, remotely available git versions for + the module to be installed and all its dependencies + + - patches -- The patches for unpublished commits and uncommitted changes + - patchModule -- The paths for the modules which has unpublished commits and uncommited changes + - patchRelPath -- The realative paths of the generated patch files + + - topFolderName -- The of the folder that the install script creates upon execution to install the module in. + If an empty string is passed, no folder will be created and installation happens in place. + - optsRelPath -- The relative path of custom opts file called by 'dunecontrol' + + """ + + with open(instFileName, 'w') as installFile: + # env and intro about script + installFile.write("##!/usr/bin/env python3\n\n") + installFile.write("'''\n") + installFile.write("This script installs the module {} together with all dependencies.\n".format(modName)) + installFile.write("'''\n") + + # import libraies + installFile.write(""" +import os +import sys +import subprocess +import traceback + +""") + + # define funtions (git utilites, installModule, print messages) + installFile.write(""" +def show_message(message): + print("*" * 120) + print(message) + print("*" * 120 + "\\n") + + +# execute a command in a folder and retrieve the output +def runCommandFromPath(command, path="."): + curPath = os.getcwd() + os.chdir(path) + subprocess.run(command) + os.chdir(curPath) + + +def git_clone(url): + clone = ["git", "clone"] + runCommandFromPath(command=[*clone, url]) + + +def git_setbranch(branch): + checkout = ["git", "checkout", branch] + runCommandFromPath(command=checkout) + + +def git_setrevision(sha): + reset = ["git", "reset", "--hard", sha] + runCommandFromPath(command=reset) + + +def git_apply_patch(folder, patch): + apply = ["git", "apply", patch] + runCommandFromPath(command=apply, path=folder) + + +def installModule(urls, deps, patches): + for url in urls: + git_clone(url) + for folder, branch, sha in zip(deps["folders"], deps["branches"], deps["shas"]): + os.chdir(folder) + git_setbranch(branch) + git_setrevision(sha) + os.chdir("..") + + for folder, patch in zip(patches["folders"], patches["files"]): + git_apply_patch(folder, patch)\n +""") + + # main function + installFile.write("\nif __name__ == '__main__':\n") + + # step 1: generate patches + unitIndentation = ' '*4 + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + "# (1/3) Generate Patches\n") + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + 'show_message("(1/3) Creating patches for unpublished commits and uncommitted changes...")\n') + + for depModPath in set(patchModule): + depModName = getModuleInfo(depModPath, "Module") + if 'unpublished' in patches[depModPath]: + patchString = str(patches[depModPath]['unpublished']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') + installFile.write(unitIndentation + "with open('{}_unpublished.patch', 'w') as patchFile:\n".format(depModName)) + installFile.write( + unitIndentation*2 + "patchFile.write(\"\"\"" + + patchString + + '\"\"\" )' + "\n" + ) + if 'uncommitted' in patches[depModPath]: + patchString = str(patches[depModPath]['uncommitted']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') + installFile.write( + unitIndentation + "with open('{}_uncommitted.patch', 'w') as patchFile:\n".format(depModName)) + installFile.write( + unitIndentation*2 + "patchFile.write(\"\"\"" + + patchString + + '\"\"\" )' + "\n" + ) + + installFile.write("\n" + unitIndentation + 'show_message("(1/3) Step completed. All patch files are generated.")\n') + + # step2: clone repositories, set branches and commits, apply patches + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + "# (2/3) Clone repositories, set branches and commits, apply patches\n") + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + 'show_message("(2/3) Cloning repositories, setting branches and commits, applying patches...")\n') + + if topFolderName: + installFile.write(' os.makedirs("./DUMUX", exist_ok=True)\n' + ' os.chdir("DUMUX")\n\n') + installFile.write(unitIndentation + 'urls = [\n') + for dep in folders: + installFile.write(unitIndentation*2 + "\"" + (versions[dep]['remote']) + "\"," + '\n') + installFile.write(unitIndentation*2 + ']\n') + + installFile.write(unitIndentation + r'deps = {}' + '\n') + + installFile.write(unitIndentation + 'deps["folders"] = [\n') + for dep in folders: + installFile.write(unitIndentation*2 + "\"" + dep + "\"," + '\n') + installFile.write(unitIndentation*2 + ']\n') + + installFile.write(unitIndentation + 'deps["branches"] = [\n') + for dep in folders: + installFile.write(unitIndentation*2 + "\"" + versions[dep]['branch'] + "\"," + '\n') + installFile.write(unitIndentation*2 + ']\n') + + installFile.write(unitIndentation + 'deps["shas"] = [\n') + for dep in folders: + installFile.write(unitIndentation*2 + "\"" + versions[dep]['revision'] + "\"," + '\n') + installFile.write(unitIndentation*2 + ']\n') + + installFile.write(unitIndentation + r'patches = {}' + '\n') + + installFile.write(unitIndentation + 'patches["folders"] = [\n') + for patchfolder in patchModule: + installFile.write(unitIndentation*2 + "\"" + patchfolder + "\"," + '\n') + installFile.write(unitIndentation*2 + ']\n') + + installFile.write(unitIndentation + 'patches["files"] = [\n') + for patchPath in patchRelPath: + if topFolderName: + patchPath = os.path.join("..", patchPath) + installFile.write(unitIndentation*2 + "\"" + patchPath + "\"," + '\n') + installFile.write(unitIndentation*2 + ']\n') + + installFile.write(unitIndentation + 'installModule(urls, deps, patches)\n') + installFile.write(unitIndentation + 'show_message("(2/3) Repositories are cloned and set properly.")\n') + + # step3: configure with dunecontrol and build tests + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + "# (3/3) Configure and build\n") + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + 'show_message("(3/3) Configure and build dune modules and dumux using dunecontrol....")\n') + installFile.write(unitIndentation + 'runCommandFromPath(command=["./dune-common/bin/dunecontrol", "--opts={}", "all"])\n'.format(optsRelPath)) + installFile.write(unitIndentation + 'os.chdir("{}/build-cmake")\n'.format(modFolder)) + installFile.write(unitIndentation + 'runCommandFromPath(command=["make buildtest"])\n') + installFile.write(unitIndentation + 'show_message("(3/3) Step completed. Succesfully configured and built tests.")\n') -- GitLab From 347c972256fa1a007fa210c9ecb7772ad69efba8 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Tue, 29 Jun 2021 18:27:46 +0000 Subject: [PATCH 27/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/makeinstallscript.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 9d71c62f89..682449c285 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -31,7 +31,8 @@ def makeInstallScript(path, topFolderName='DUMUX', optsFile=None, skipFolders=None, - suppressHints=False): + suppressHints=False + language="bash"): cwd = os.getcwd() modPath = os.path.abspath(os.path.join(cwd, path)) -- GitLab From 678ca085acb8c4d306ad00c2c6f1eaa13d160a21 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Tue, 29 Jun 2021 18:31:39 +0000 Subject: [PATCH 28/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index 23e5facaa8..f104242e5d 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -247,7 +247,7 @@ def writeShellInstallScript(instFileName, for depModPath in set(patchModule): depModName = getModuleInfo(depModPath, "Module") if 'unpublished' in patches[depModPath]: - installFile.write("cat >> {}_unpublished.patch <<'EOF'\n".format(depModName) + installFile.write(f"cat >> {depModName}/unpublished.patch <<'EOF'" + patches[depModPath]['unpublished'] + "EOF\n") if 'uncommitted' in patches[depModPath]: -- GitLab From 5a20fef37145fa683dc5c8b1eb28019b4b0efb57 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 30 Jun 2021 00:57:28 +0200 Subject: [PATCH 29/47] [bin][extractmodule] Use dumux as example root folder name in Installation instruction --- bin/extractmodule/extract_as_new_module.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 590c669bd0..53c9f0b810 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -187,18 +187,18 @@ def info_readme_installation(remoteurl, install_script_name, new_module_name): ## Installation The installation procedure is done as follows : -Create a root folder, e.g. `$NewFolder$`, enter the previously created folder +Create a root folder, e.g. `DUMUX`, enter the previously created folder, clone the remote and use the install script `{install_script_name}` provided in this repository to install all dependent modules. ```sh -mkdir $NewFolder$ -cd $NewFolder$ +mkdir DUMUX +cd DUMUX git clone {remoteurl} ./{new_module_name}/{install_script_name} ``` -This will clone all modules into your created folder `$NewFolder$`, +This will clone all modules into the directory `DUMUX`, configure your module with `dunecontrol` and build tests. """ -- GitLab From 024b5718e0e1366307355c269974548a8f819769 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 30 Jun 2021 22:40:28 +0200 Subject: [PATCH 30/47] [bin] Put patch in respective folder and resulting simplification/reconstruction of script --- bin/extractmodule/makeinstallscript.py | 6 +- bin/extractmodule/util.py | 117 ++++++++++++------------- 2 files changed, 61 insertions(+), 62 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 682449c285..bc0621d646 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -31,7 +31,7 @@ def makeInstallScript(path, topFolderName='DUMUX', optsFile=None, skipFolders=None, - suppressHints=False + suppressHints=False, language="bash"): cwd = os.getcwd() @@ -112,9 +112,9 @@ def makeInstallScript(path, depModName = getModuleInfo(depModPath, "Module") if 'unpublished' in patches[depModPath]: patchModule.append(depModPath) - patchRelPath.append(os.path.relpath("{}_unpublished.patch".format(depModName), depModPath)) + patchRelPath.append(os.path.relpath("{}/unpublished.patch".format(depModName), depModPath)) if 'uncommitted' in patches[depModPath]: - patchRelPath.append(os.path.relpath("{}_uncommitted.patch".format(depModName), depModPath)) + patchRelPath.append(os.path.relpath("{}/uncommitted.patch".format(depModName), depModPath)) patchModule.append(depModPath) writeShellInstallScript(instFileName, diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index f104242e5d..643e0b6052 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -8,7 +8,6 @@ try: sys.path.append(os.path.join(path, '../util')) from common import callFromPath, runCommand - from getmoduleinfo import getModuleInfo except Exception: sys.exit('Could not import common module') @@ -177,15 +176,13 @@ def writeShellInstallScript(instFileName, # write function to install the new module and its dependencies into install script installFile.write( - '# defines the function to install module with urls, shas and patches\n' + '# defines function to clone modules, set branches and commits\n' 'installModule()\n' '{\n' ' URLS=$1\n' ' DEPFOLDERS=$2\n' ' DEPBRANCHES=$3\n' - ' DEPSHAS=$4\n' - ' PATCHES=$5\n' - ' PATCHFOLDERS=$6\n\n' + ' DEPSHAS=$4\n\n' ' for url in ${URLS[@]}; do\n' ' if ! git clone $url; then\n' ' echo "--Error: failed to clone $url"\n' @@ -202,11 +199,17 @@ def writeShellInstallScript(instFileName, ' echo "-- Error: failed to check out commit ${DEPSHAS[index]} in module ${DEPFOLDERS[index]}."\n' ' fi\n' ' cd ..\n' - ' done\n\n' + ' done\n' + '}\n\n' + + '# defines function to apply patches\n' + 'applyPatch()\n' + '{\n' + ' PATCHES=$1\n' + ' PATCHFOLDERS=$2\n\n' ' for i in ${!PATCHES[@]}; do\n' ' cd ${PATCHFOLDERS[i]}\n' - f' if ! git apply {"../" if topFolderName else ""}$' - '{PATCHES[i]}; then\n' + ' if ! git apply ${PATCHES[i]}; then\n' ' echo "--Error: failed to apply patch $patch"\n' ' fi\n' ' cd ..\n' @@ -244,17 +247,6 @@ def writeShellInstallScript(instFileName, installFile.write('\n'.join(patchRelPath)) installFile.write('\n)\n\n') - for depModPath in set(patchModule): - depModName = getModuleInfo(depModPath, "Module") - if 'unpublished' in patches[depModPath]: - installFile.write(f"cat >> {depModName}/unpublished.patch <<'EOF'" - + patches[depModPath]['unpublished'] - + "EOF\n") - if 'uncommitted' in patches[depModPath]: - installFile.write("cat >> {}_uncommitted.patch <<'EOF'\n".format(depModName) - + patches[depModPath]['uncommitted'] - + "EOF\n") - def writeCommandWithErrorCheck(command, errorMessage, indentationLevel=0): unitIndentation = ' '*4 indent = unitIndentation*indentationLevel @@ -291,7 +283,17 @@ def writeShellInstallScript(instFileName, ' script is executed.\n\n' ) - installFile.write('installModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS $PATCHES $PATCHFOLDERS\n\n') + installFile.write('installModule $URLS $DEPFOLDERS $DEPBRANCHES $DEPSHAS\n\n') + for depModPath in set(patchModule): + if 'unpublished' in patches[depModPath]: + installFile.write(f"cat >> {depModPath}/unpublished.patch <<'EOF'\n" + + patches[depModPath]['unpublished'] + + "EOF\n") + if 'uncommitted' in patches[depModPath]: + installFile.write(f"cat >> {depModPath}/uncommitted.patch <<'EOF'\n" + + patches[depModPath]['uncommitted'] + + "EOF\n") + installFile.write('applyPatch $PATCHES $PATCHFOLDERS\n\n') # write configure command installFile.write('echo "-- All modules haven been cloned successfully. ' @@ -400,7 +402,7 @@ def git_apply_patch(folder, patch): runCommandFromPath(command=apply, path=folder) -def installModule(urls, deps, patches): +def installModule(urls, deps): for url in urls: git_clone(url) for folder, branch, sha in zip(deps["folders"], deps["branches"], deps["shas"]): @@ -409,47 +411,20 @@ def installModule(urls, deps, patches): git_setrevision(sha) os.chdir("..") + +def applyPatch(patches): for folder, patch in zip(patches["folders"], patches["files"]): git_apply_patch(folder, patch)\n """) - # main function + # write main function and step1: clone repositories, set branches and commits installFile.write("\nif __name__ == '__main__':\n") - # step 1: generate patches unitIndentation = ' '*4 installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + "# (1/3) Generate Patches\n") - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + 'show_message("(1/3) Creating patches for unpublished commits and uncommitted changes...")\n') - - for depModPath in set(patchModule): - depModName = getModuleInfo(depModPath, "Module") - if 'unpublished' in patches[depModPath]: - patchString = str(patches[depModPath]['unpublished']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') - installFile.write(unitIndentation + "with open('{}_unpublished.patch', 'w') as patchFile:\n".format(depModName)) - installFile.write( - unitIndentation*2 + "patchFile.write(\"\"\"" - + patchString - + '\"\"\" )' + "\n" - ) - if 'uncommitted' in patches[depModPath]: - patchString = str(patches[depModPath]['uncommitted']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') - installFile.write( - unitIndentation + "with open('{}_uncommitted.patch', 'w') as patchFile:\n".format(depModName)) - installFile.write( - unitIndentation*2 + "patchFile.write(\"\"\"" - + patchString - + '\"\"\" )' + "\n" - ) - - installFile.write("\n" + unitIndentation + 'show_message("(1/3) Step completed. All patch files are generated.")\n') - - # step2: clone repositories, set branches and commits, apply patches - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + "# (2/3) Clone repositories, set branches and commits, apply patches\n") + installFile.write(unitIndentation + "# (1/3) Clone repositories, set branches and commits\n") installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + 'show_message("(2/3) Cloning repositories, setting branches and commits, applying patches...")\n') + installFile.write(unitIndentation + 'show_message("(1/3) Cloning repositories, setting branches and commits...")\n') if topFolderName: installFile.write(' os.makedirs("./DUMUX", exist_ok=True)\n' @@ -475,9 +450,36 @@ def installModule(urls, deps, patches): for dep in folders: installFile.write(unitIndentation*2 + "\"" + versions[dep]['revision'] + "\"," + '\n') installFile.write(unitIndentation*2 + ']\n') + installFile.write(unitIndentation + 'installModule(urls, deps)\n') + installFile.write(unitIndentation + 'show_message("(1/3) Repositories are cloned and set properly.")\n') - installFile.write(unitIndentation + r'patches = {}' + '\n') + # step 2: generate and apply patches + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + "# (2/3) Generate and apply patches\n") + installFile.write(unitIndentation + "#"*80 + "\n") + installFile.write(unitIndentation + 'show_message("(2/3) Creating patches for unpublished commits and uncommitted changes...")\n') + + for depModPath in set(patchModule): + if 'unpublished' in patches[depModPath]: + patchString = str(patches[depModPath]['unpublished']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') + installFile.write(unitIndentation + "with open('{}/unpublished.patch', 'w') as patchFile:\n".format(depModPath)) + installFile.write( + unitIndentation*2 + "patchFile.write(\"\"\"" + + patchString + + '\"\"\" )' + "\n" + ) + if 'uncommitted' in patches[depModPath]: + patchString = str(patches[depModPath]['uncommitted']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') + installFile.write( + unitIndentation + "with open('{}/uncommitted.patch', 'w') as patchFile:\n".format(depModPath)) + installFile.write( + unitIndentation*2 + "patchFile.write(\"\"\"" + + patchString + + '\"\"\" )' + "\n" + ) + installFile.write(unitIndentation + 'show_message("(2/3) Applying patches for unpublished commits and uncommitted changes...")\n') + installFile.write(unitIndentation + r'patches = {}' + '\n') installFile.write(unitIndentation + 'patches["folders"] = [\n') for patchfolder in patchModule: installFile.write(unitIndentation*2 + "\"" + patchfolder + "\"," + '\n') @@ -485,13 +487,10 @@ def installModule(urls, deps, patches): installFile.write(unitIndentation + 'patches["files"] = [\n') for patchPath in patchRelPath: - if topFolderName: - patchPath = os.path.join("..", patchPath) installFile.write(unitIndentation*2 + "\"" + patchPath + "\"," + '\n') installFile.write(unitIndentation*2 + ']\n') - - installFile.write(unitIndentation + 'installModule(urls, deps, patches)\n') - installFile.write(unitIndentation + 'show_message("(2/3) Repositories are cloned and set properly.")\n') + installFile.write(unitIndentation + 'applyPatch(patches)\n') + installFile.write("\n" + unitIndentation + 'show_message("(2/3) Step completed. All patch files are generated and applied.")\n') # step3: configure with dunecontrol and build tests installFile.write(unitIndentation + "#"*80 + "\n") -- GitLab From a39dcb3d75ebe8641eb0b90668cb0f401d09029d Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 30 Jun 2021 23:04:01 +0200 Subject: [PATCH 31/47] [bin][util] Use multiple lines string and shorter name for unitIndentation --- bin/extractmodule/util.py | 102 ++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index 643e0b6052..dbe74b4975 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -418,86 +418,90 @@ def applyPatch(patches): """) # write main function and step1: clone repositories, set branches and commits - installFile.write("\nif __name__ == '__main__':\n") - - unitIndentation = ' '*4 - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + "# (1/3) Clone repositories, set branches and commits\n") - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + 'show_message("(1/3) Cloning repositories, setting branches and commits...")\n') + indent = ' '*4 + installFile.write(""" +if __name__ == '__main__': + ########################################################################### + # (1/3) Clone repositories, set branches and commits + ########################################################################### + show_message("(1/3) Cloning repositories, setting branches and commits...") +""") if topFolderName: installFile.write(' os.makedirs("./DUMUX", exist_ok=True)\n' ' os.chdir("DUMUX")\n\n') - installFile.write(unitIndentation + 'urls = [\n') + installFile.write(indent + 'urls = [\n') for dep in folders: - installFile.write(unitIndentation*2 + "\"" + (versions[dep]['remote']) + "\"," + '\n') - installFile.write(unitIndentation*2 + ']\n') + installFile.write(indent*2 + "\"" + (versions[dep]['remote']) + "\"," + '\n') + installFile.write(indent*2 + ']\n') - installFile.write(unitIndentation + r'deps = {}' + '\n') + installFile.write(indent + r'deps = {}' + '\n') - installFile.write(unitIndentation + 'deps["folders"] = [\n') + installFile.write(indent + 'deps["folders"] = [\n') for dep in folders: - installFile.write(unitIndentation*2 + "\"" + dep + "\"," + '\n') - installFile.write(unitIndentation*2 + ']\n') + installFile.write(indent*2 + "\"" + dep + "\"," + '\n') + installFile.write(indent*2 + ']\n') - installFile.write(unitIndentation + 'deps["branches"] = [\n') + installFile.write(indent + 'deps["branches"] = [\n') for dep in folders: - installFile.write(unitIndentation*2 + "\"" + versions[dep]['branch'] + "\"," + '\n') - installFile.write(unitIndentation*2 + ']\n') + installFile.write(indent*2 + "\"" + versions[dep]['branch'] + "\"," + '\n') + installFile.write(indent*2 + ']\n') - installFile.write(unitIndentation + 'deps["shas"] = [\n') + installFile.write(indent + 'deps["shas"] = [\n') for dep in folders: - installFile.write(unitIndentation*2 + "\"" + versions[dep]['revision'] + "\"," + '\n') - installFile.write(unitIndentation*2 + ']\n') - installFile.write(unitIndentation + 'installModule(urls, deps)\n') - installFile.write(unitIndentation + 'show_message("(1/3) Repositories are cloned and set properly.")\n') + installFile.write(indent*2 + "\"" + versions[dep]['revision'] + "\"," + '\n') + installFile.write(indent*2 + ']\n') + installFile.write(indent + 'installModule(urls, deps)\n') + installFile.write(indent + 'show_message("(1/3) Repositories are cloned and set properly.")') # step 2: generate and apply patches - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + "# (2/3) Generate and apply patches\n") - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + 'show_message("(2/3) Creating patches for unpublished commits and uncommitted changes...")\n') - + installFile.write(""" + ########################################################################### + # (2/3) Generate and apply patches + ########################################################################### + show_message("(2/3) Creating patches for unpublished commits and uncommitted changes...") +""") for depModPath in set(patchModule): if 'unpublished' in patches[depModPath]: patchString = str(patches[depModPath]['unpublished']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') - installFile.write(unitIndentation + "with open('{}/unpublished.patch', 'w') as patchFile:\n".format(depModPath)) + installFile.write(indent + "with open('{}/unpublished.patch', 'w') as patchFile:\n".format(depModPath)) installFile.write( - unitIndentation*2 + "patchFile.write(\"\"\"" + indent*2 + "patchFile.write(\"\"\"" + patchString + '\"\"\" )' + "\n" ) if 'uncommitted' in patches[depModPath]: patchString = str(patches[depModPath]['uncommitted']).replace('\\', r'\\').replace("'", "\\'").replace('"', '\\"') installFile.write( - unitIndentation + "with open('{}/uncommitted.patch', 'w') as patchFile:\n".format(depModPath)) + indent + "with open('{}/uncommitted.patch', 'w') as patchFile:\n".format(depModPath)) installFile.write( - unitIndentation*2 + "patchFile.write(\"\"\"" + indent*2 + "patchFile.write(\"\"\"" + patchString + '\"\"\" )' + "\n" ) - installFile.write(unitIndentation + 'show_message("(2/3) Applying patches for unpublished commits and uncommitted changes...")\n') - installFile.write(unitIndentation + r'patches = {}' + '\n') - installFile.write(unitIndentation + 'patches["folders"] = [\n') + installFile.write(indent + 'show_message("(2/3) Applying patches for unpublished commits and uncommitted changes...")\n') + installFile.write(indent + r'patches = {}' + '\n') + installFile.write(indent + 'patches["folders"] = [\n') for patchfolder in patchModule: - installFile.write(unitIndentation*2 + "\"" + patchfolder + "\"," + '\n') - installFile.write(unitIndentation*2 + ']\n') + installFile.write(indent*2 + "\"" + patchfolder + "\"," + '\n') + installFile.write(indent*2 + ']\n') - installFile.write(unitIndentation + 'patches["files"] = [\n') + installFile.write(indent + 'patches["files"] = [\n') for patchPath in patchRelPath: - installFile.write(unitIndentation*2 + "\"" + patchPath + "\"," + '\n') - installFile.write(unitIndentation*2 + ']\n') - installFile.write(unitIndentation + 'applyPatch(patches)\n') - installFile.write("\n" + unitIndentation + 'show_message("(2/3) Step completed. All patch files are generated and applied.")\n') + installFile.write(indent*2 + "\"" + patchPath + "\"," + '\n') + installFile.write(indent*2 + ']\n') + installFile.write(indent + 'applyPatch(patches)\n') + installFile.write("\n" + indent + 'show_message("(2/3) Step completed. All patch files are generated and applied.")') # step3: configure with dunecontrol and build tests - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + "# (3/3) Configure and build\n") - installFile.write(unitIndentation + "#"*80 + "\n") - installFile.write(unitIndentation + 'show_message("(3/3) Configure and build dune modules and dumux using dunecontrol....")\n') - installFile.write(unitIndentation + 'runCommandFromPath(command=["./dune-common/bin/dunecontrol", "--opts={}", "all"])\n'.format(optsRelPath)) - installFile.write(unitIndentation + 'os.chdir("{}/build-cmake")\n'.format(modFolder)) - installFile.write(unitIndentation + 'runCommandFromPath(command=["make buildtest"])\n') - installFile.write(unitIndentation + 'show_message("(3/3) Step completed. Succesfully configured and built tests.")\n') + installFile.write(""" + ########################################################################### + # (3/3) Configure and build + ########################################################################### + show_message("(3/3) Configure and build dune modules and dumux using dunecontrol....") + runCommandFromPath(command=["./dune-common/bin/dunecontrol", "--opts={0}", "all"]) + os.chdir("{1}/build-cmake") + runCommandFromPath(command=["make buildtest"]) + show_message("(3/3) Step completed. Succesfully configured and built tests.") +""".format(*[optsRelPath, modFolder])) -- GitLab From 395fc3322f4a3c1f881ab18e8f01638752a974cb Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 30 Jun 2021 23:06:15 +0200 Subject: [PATCH 32/47] [util] Remove unneccesary python library and small fixes (squashed later) --- bin/extractmodule/util.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index dbe74b4975..1e9c9a4768 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -297,9 +297,9 @@ def writeShellInstallScript(instFileName, # write configure command installFile.write('echo "-- All modules haven been cloned successfully. ' - 'Configuring project..."\n') + 'Configuring project..."\n') writeCommandWithErrorCheck( - './dune-common/bin/dunecontrol --opts=dumux/cmake.opts all'.format(optsRelPath), + './dune-common/bin/dunecontrol --opts={} all'.format(optsRelPath), '--Error: could not configure project' ) @@ -360,9 +360,7 @@ def writePythonInstallScript(instFileName, # import libraies installFile.write(""" import os -import sys import subprocess -import traceback """) @@ -502,6 +500,6 @@ if __name__ == '__main__': show_message("(3/3) Configure and build dune modules and dumux using dunecontrol....") runCommandFromPath(command=["./dune-common/bin/dunecontrol", "--opts={0}", "all"]) os.chdir("{1}/build-cmake") - runCommandFromPath(command=["make buildtest"]) + runCommandFromPath(command=["make", "build_tests"]) show_message("(3/3) Step completed. Succesfully configured and built tests.") """.format(*[optsRelPath, modFolder])) -- GitLab From e8bbf77286cd1c010c59d1e490c3f9da3dad26c5 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Thu, 1 Jul 2021 00:47:25 +0200 Subject: [PATCH 33/47] [bin][makeinstall] Add option for user to choose python or bash and Correct the responding output --- bin/extractmodule/makeinstallscript.py | 56 +++++++++++++++++--------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index bc0621d646..a5434914d5 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -12,6 +12,7 @@ from util import getPersistentVersions from util import printVersionTable from util import getPatches from util import writeShellInstallScript +from util import writePythonInstallScript try: path = os.path.split(os.path.abspath(__file__))[0] @@ -25,6 +26,21 @@ if sys.version_info[0] < 3: sys.exit("\nError': Python3 required") +# ask user to speficy the language of the generated script +def python_or_bash(): + inp = input('In python or bash do you want to generate' + 'the install script?[p/b] (p:"Python", b:"Bash")') + if inp == "b" or inp == "sh" or inp =="bash" or inp == "shell": + print("You choose bash as language for install script.") + return "bash" + elif inp == "p" or inp == "py" or inp == "python": + print("You choose python as language for install script") + return "python" + else: + print("--Error: Unknown type. You must choose between python or bash!") + return python_or_bash() + + def makeInstallScript(path, fileName=None, ignoreUntracked=False, @@ -46,7 +62,7 @@ def makeInstallScript(path, " module path is correct and contains 'dune.module'.") if not fileName: - instFileName = 'install_' + modName + '.sh' + instFileName = 'install_' + modName + '.%s'%("sh" if language == "bash" else "py") else: instFileName = fileName print("\n-- Creating install script '{}' for module '{}' in folder '{}'" @@ -89,10 +105,10 @@ def makeInstallScript(path, print(" on top of which we will generate the required patches") printVersionTable(versions) - print("\n-- Creating patches for unpublished commits and uncommitted changes") + print("\n-- Determining patches information for unpublished commits and uncommited changes") patches = getPatches(versions) - # write installation shell script (switch to relative paths) + # write installation script (switch to relative paths) versions = {os.path.relpath(p, modParentPath): v for p, v in versions.items()} patches = {os.path.relpath(p, modParentPath): v for p, v in patches.items()} @@ -117,22 +133,21 @@ def makeInstallScript(path, patchRelPath.append(os.path.relpath("{}/uncommitted.patch".format(depModName), depModPath)) patchModule.append(depModPath) - writeShellInstallScript(instFileName, - modName, modFolder, - folders, versions, - patches, patchModule, patchRelPath, - topFolderName, optsRelPath) + argsGenerateScript = (instFileName, + modName, modFolder, folders, versions, + patches, patchModule, patchRelPath, + topFolderName, optsRelPath) - print("\n-- Successfully created install script file " + instFileName) - - subprocess.call(['chmod', 'u+x', instFileName]) # make script executable + if language == "bash": + writeShellInstallScript(*argsGenerateScript) + else: + writePythonInstallScript(*argsGenerateScript) + print("\n"+"*"*120) + print("-- Successfully created install script file " + instFileName) + print("*"*120) - if len(patches) > 0: - if not suppressHints: - print("-> You should now commit and publish the 'patches' folder and this install script in your module such that others can use it.\n" - " IMPORTANT: After you committed the patches, you have to adjust the line of the install script in which your module is checked out to a specific commit.\n" - " That is, in the line 'git reset --hard COMMIT_SHA' for your module, replace COMMIT_SHA by the commit in which you added the patches.\n" - " If patches had to be created for your own module, please think about comitting and pushing your local changes and rerunning this script again.") + if language == "bash": + subprocess.call(['chmod', 'u+x', instFileName]) # make script executable if not suppressHints: print(f"\n-- You might want to put installation instructions into the README.md file of your module, for instance:\n" @@ -142,7 +157,7 @@ def makeInstallScript(path, f"\n" f" ```sh\n" f" wget {versions[modFolder]['remote']}/{instFileName}\n" - f" bash {instFileName}\n" + f" {'bash' if language == 'bash' else 'python3'} {instFileName}\n" f" ```\n") if topFolderName: @@ -192,6 +207,8 @@ if __name__ == '__main__': cmdArgs = vars(parser.parse_args()) + scriptLanguage = python_or_bash() + makeInstallScript( path=cmdArgs['path'], fileName=cmdArgs.get('filename', None), @@ -199,5 +216,6 @@ if __name__ == '__main__': topFolderName=cmdArgs.get('topfoldername', None), optsFile=cmdArgs.get('optsFile', None), skipFolders=cmdArgs.get('skipfolders', None), - suppressHints=cmdArgs.get('suppresshints', False) + suppressHints=cmdArgs.get('suppresshints', False), + language=scriptLanguage ) -- GitLab From 342f9c47a703d613f09ee1c7bc039e8d88fb9340 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Thu, 1 Jul 2021 19:45:55 +0200 Subject: [PATCH 34/47] [bin][makeInstall] Make script language an arg user must choose if not specify correctly --- bin/extractmodule/makeinstallscript.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index a5434914d5..70449c76d9 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -30,7 +30,7 @@ if sys.version_info[0] < 3: def python_or_bash(): inp = input('In python or bash do you want to generate' 'the install script?[p/b] (p:"Python", b:"Bash")') - if inp == "b" or inp == "sh" or inp =="bash" or inp == "shell": + if inp == "b" or inp == "sh" or inp == "bash" or inp == "shell": print("You choose bash as language for install script.") return "bash" elif inp == "p" or inp == "py" or inp == "python": @@ -48,7 +48,10 @@ def makeInstallScript(path, optsFile=None, skipFolders=None, suppressHints=False, - language="bash"): + language=None): + + if language != "bash" and language != "python": + language = python_or_bash() cwd = os.getcwd() modPath = os.path.abspath(os.path.join(cwd, path)) @@ -205,9 +208,12 @@ if __name__ == '__main__': required=False, default=False, help='if output needs to be suppressed') - cmdArgs = vars(parser.parse_args()) + parser.add_argument('-l', '--language', + required=False, default=None, + help='Language used to write install script, ' + 'currently only python and bash are supported') - scriptLanguage = python_or_bash() + cmdArgs = vars(parser.parse_args()) makeInstallScript( path=cmdArgs['path'], @@ -217,5 +223,5 @@ if __name__ == '__main__': optsFile=cmdArgs.get('optsFile', None), skipFolders=cmdArgs.get('skipfolders', None), suppressHints=cmdArgs.get('suppresshints', False), - language=scriptLanguage + language=cmdArgs.get('language', None) ) -- GitLab From 7155bdacf90373701ae6da6adc72b32466ef23df Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Thu, 1 Jul 2021 19:47:33 +0200 Subject: [PATCH 35/47] [bin][extractmodule] Support python install script for extracted module and adapt the README and output --- bin/extractmodule/extract_as_new_module.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 53c9f0b810..cfaed64675 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -18,6 +18,7 @@ from functools import partial from util import getPersistentVersions from util import versionTable from makeinstallscript import makeInstallScript +from makeinstallscript import python_or_bash try: path = os.path.split(os.path.abspath(__file__))[0] sys.path.append(os.path.join(path, '../bin/util')) @@ -181,7 +182,7 @@ please go to the build folders corresponding to the sources listed above.\n ################################################################### # Installation part of README.md ################################################################### -def info_readme_installation(remoteurl, install_script_name, new_module_name): +def info_readme_installation(remoteurl, install_script_name, new_module_name, language): return f""" ## Installation @@ -195,7 +196,7 @@ provided in this repository to install all dependent modules. mkdir DUMUX cd DUMUX git clone {remoteurl} -./{new_module_name}/{install_script_name} +{"./" if language == "bash" else "python3 "}{new_module_name}/{install_script_name} ``` This will clone all modules into the directory `DUMUX`, @@ -424,7 +425,7 @@ if __name__ == "__main__": # ask user if to write version information into README.md if query_yes_no("Write detailed version information" - " (folder/branch/commits/dates) into README.md?"): + " (folder/branch/commits/dates) into README.md?\n"): print("Looking for the dune modules in path: " + os.path.abspath(".") + "...") @@ -442,11 +443,12 @@ if __name__ == "__main__": # if there is a remote repository available # we can directly push the source code and - # also create a "one-click" installation script - install_script_name = 'install_' + new_module_name + '.sh' + # also create a "one-click" installation scrip + language = python_or_bash() + install_script_name = 'install_' + new_module_name + '.%s'%("sh" if language == "bash" else "py") try: makeInstallScript(new_module_path, ignoreUntracked=True, skipFolders=new_module_name, - suppressHints=True, topFolderName=None) + suppressHints=True, topFolderName=None, language=language) except Exception: sys.exit( f"Automatically generate install script {install_script_name} failed." @@ -464,7 +466,7 @@ if __name__ == "__main__": # append install information into readme with open(readme_path, "a") as readme_file: - readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name)) + readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name, language)) run_from_mod('git init') run_from_mod('git add .') run_from_mod('git commit -m "Initial commit"') @@ -473,12 +475,12 @@ if __name__ == "__main__": # output guidance for users to create install script manually else: - remoteurl = "{$remoteurl$} ({$remoteurl$} is the URL for your remote git repository)" + remoteurl = "{$remoteurl$} (needs to be manuelly adapted later)" # append install information into readme with open(readme_path, "a") as readme_file: - readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name)) + readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name, language)) run_from_mod('git init') run_from_mod('git add .') run_from_mod('git commit -m "Initial commit"') - print("Please remember to replace the placeholder $remoteurl$ in README installation\n" - "with the correct URL after your remote git repository is created.") + print("\nPlease remember to replace placeholder $remoteurl$ in installation part\n" + "of the README file with the correct URL of your remote git repository.") -- GitLab From 793ab6a5050630d4fb922e36d816218bcd811512 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Fri, 2 Jul 2021 00:34:27 +0200 Subject: [PATCH 36/47] [bin][extractmodule] Fix typo. Kudos Yue --- bin/extractmodule/extract_as_new_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index cfaed64675..005ebbc0cc 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -141,7 +141,7 @@ the module '{module_dir}': {os.linesep.join([f" - {os.path.relpath(f, module_path)}" for f in subfolders])} and all headers contained in '{module_dir}' -tha are required to build the exectutables from the sources: +that are required to build the exectutables from the sources: {os.linesep.join([f" - {s}" for s in source_files])} The extracted files are copied into a new DUNE module -- GitLab From 4e7cb71c6721498e4a7e0b5c8660ee445f6ef5f2 Mon Sep 17 00:00:00 2001 From: Yue Wang Date: Fri, 2 Jul 2021 08:53:16 +0000 Subject: [PATCH 37/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/extract_as_new_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 005ebbc0cc..0f995f3ff0 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -311,7 +311,7 @@ if __name__ == "__main__": ) # get the base path of the new module - new_module_path = module_path.replace(module_dir, new_module_name, 1) + new_module_path = new_module_name.join(module_path.rsplit(module_dir,1)) # copy the source tree # copy all base folders complete, then delete the unnecessary ones -- GitLab From 3caf68bca3d30a2fc84325dacd9210e5901d626a Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Fri, 2 Jul 2021 11:33:18 +0200 Subject: [PATCH 38/47] [bin][extractmodule] Separate path_check from extract_source and main, Make it correct/robust --- bin/extractmodule/extract_as_new_module.py | 23 +++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 0f995f3ff0..afd17c2e16 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -102,6 +102,20 @@ def get_remote_url(repo_path): sys.stdout.write("ERROR: The remote reposity is not empty!.\n") +def path_check(basedir, subdir): + # check if basedir contained in the script path + if not os.path.isdir(basedir): + sys.exit("ERROR: You need to run the script" + f"one level above the folder {basedir}.\n" + f"Run \"{os.path.basename(__file__)} --help\" for details.") + + # check if subdir immediate subfolder of basedir + list_subdir = [subdir.path for subdir in os.scandir(basedir) if subdir.is_dir()] + for folder in subdir: + if os.path.join(basedir, folder.strip(os.sep)) not in list_subdir: + raise NameError(f"Subfolder '{folder}' is not a subfolder of '{basedir}'") + + def extract_sources_files(module_dir, subfolders): def find_files_with_pattern(pattern, path): result = [] @@ -114,8 +128,6 @@ def extract_sources_files(module_dir, subfolders): sources = [] for folder in subfolders: folder_path = os.path.join(module_dir, folder) - if not os.path.isdir(folder_path): - raise NameError(f"Subfolder '{folder}' is not a subfolder of '{module_dir}'") sources += find_files_with_pattern("*.cc", os.path.abspath(folder_path)) return sources @@ -261,11 +273,8 @@ if __name__ == "__main__": # make unique set of subfolders subfolders = list(set(args['subfolder'])) - # check if we are above module_dir - if not os.path.isdir(module_dir): - sys.exit("ERROR: You need to run the script" - f"one level above the folder {module_dir}.\n" - f"Run \"{os.path.basename(__file__)} --help\" for details.") + # check paths to prevenet possible errors + path_check(module_dir, subfolders) # determine all source files in the paths passed as arguments source_files = extract_sources_files(module_dir, subfolders) -- GitLab From f129dc37960cfb56fcda9058c0c5a43fb863fa92 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Sun, 4 Jul 2021 22:28:42 +0200 Subject: [PATCH 39/47] [bin][extractmodule] Add yes to all option, change default and correct output --- bin/extractmodule/extract_as_new_module.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index afd17c2e16..74f5666f62 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -227,8 +227,6 @@ def info_make_install(new_module_name): The extracted module is contained in the subfolder '{new_module_name}'. You can configure it with ./dune-common/bin/dunecontrol --opts=dumux/cmake.opts —only={new_module_name} all -To create an install script use the "makeinstallscript.py" script in the same folder. -You can run 'python3 makeinstallscript.py --help' for more information """ @@ -360,16 +358,28 @@ if __name__ == "__main__": return True return False + no_source_folder = [] for path, dirs, files in os.walk(os.path.join(new_module_path, b)): for d in dirs: dir_path = os.path.join(path, d) keep = matching_path(dir_path, new_source_files) if not keep: rel_dir_path = os.path.relpath(dir_path, new_module_path) + no_source_folder.append(rel_dir_path) + if len(no_source_folder) > 1: + no_source_lists = ('\n'.join(dir for dir in no_source_folder)) + yes_to_all = query_yes_no( + "No source files found in the following directories:\n{}\n" + "Copy them all to the new module?\n" + "(Otherwise you need to configure each folder maneully)" + .format(no_source_lists) + ) + if not yes_to_all or len(no_source_folder) < 2: + for rel_dir_path in no_source_folder: answer_is_yes = query_yes_no( f"{rel_dir_path} does not contain source files." " Copy to new module?", - default="no" + default="yes" ) if not answer_is_yes: # remove copy of directory -- GitLab From 0c02b0bb3ebc03eaaa8311cb0494efaf3334e168 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Mon, 5 Jul 2021 00:26:53 +0200 Subject: [PATCH 40/47] [bin][extractmodule] Generate log file which can be shown with DEBUG/ERROR/INFO mode to control whole process --- bin/extractmodule/extract_as_new_module.py | 86 +++++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 74f5666f62..66592b4433 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -26,6 +26,7 @@ try: from common import callFromPath, runCommand except Exception: sys.exit('Could not import common modul or getModuleInfo') +import logging # return the list of included headers including the header itself @@ -99,12 +100,14 @@ def get_remote_url(repo_path): elif (check_remote_repo == ''): return remoteurl else: + logging.error("Remote repository is not empty!") sys.stdout.write("ERROR: The remote reposity is not empty!.\n") def path_check(basedir, subdir): # check if basedir contained in the script path if not os.path.isdir(basedir): + logging.error(f"No {basedir} found in your path where the script is running!") sys.exit("ERROR: You need to run the script" f"one level above the folder {basedir}.\n" f"Run \"{os.path.basename(__file__)} --help\" for details.") @@ -113,6 +116,7 @@ def path_check(basedir, subdir): list_subdir = [subdir.path for subdir in os.scandir(basedir) if subdir.is_dir()] for folder in subdir: if os.path.join(basedir, folder.strip(os.sep)) not in list_subdir: + logging.error(f"Subfolder '{folder}' is not a subfolder of '{basedir}'") raise NameError(f"Subfolder '{folder}' is not a subfolder of '{basedir}'") @@ -140,6 +144,7 @@ def check_module(module_name): raise Exception(f"Invalid dune.module in {module_name}") except OSError: print("Could not find new Dune module. Aborting") + logging.error("The module you created is not dune-module!") raise @@ -247,6 +252,10 @@ if __name__ == "__main__": (extracts the subfolders appl and test from the module dumux-fracture) ''' + # generate a log file to control all the process + logging.basicConfig(filename="extractmodulepart.log", filemode='w', level=logging.DEBUG) + logging.info("The main function is going to be executed, preparing to take off.") + logging.debug("Passing arguments provided by user...") parser = argparse.ArgumentParser( prog='extract_as_new_module.py', usage='./dumux/bin/extractmodule/extract_as_new_module.py' @@ -264,6 +273,7 @@ if __name__ == "__main__": help='subfolder(s) of module_dir that you want to extract' ) args = vars(parser.parse_args()) + logging.debug("Parameters from terminal are passed.") # if module_dir ends with slash(es) remove it/them module_dir = args['module_dir'].strip(os.sep) @@ -272,23 +282,29 @@ if __name__ == "__main__": subfolders = list(set(args['subfolder'])) # check paths to prevenet possible errors + logging.debug("Checking if base module and subfolders exist...") path_check(module_dir, subfolders) + logging.debug("Path check is done.") # determine all source files in the paths passed as arguments source_files = extract_sources_files(module_dir, subfolders) # check if sources have been obtained + logging.debug("Checking if sources have been obtained...") if not source_files: + logging.error("No sources found in the subfolders you provide!") sys.exit( "ERROR: No source files *.cc found in the subfolders: " + ", ".join([str(x) for x in subfolders]) + ".\n" "Be sure to provide a list of paths as arguments.\n" f"Run '{os.path.basename(__file__)} --help' for details." ) + logging.debug("Sources are found in subfolders.") # try to find the duneproject script dune_project = shutil.which('duneproject', path="dune-common/bin") if dune_project is None: + logging.error("No duneproject found in dune-common/bin!") sys.exit( "ERROR: Could not find duneproject.\n" "Make sure to have duneproject in dune-common/bin" @@ -298,30 +314,44 @@ if __name__ == "__main__": print(info_explanations( module_dir, module_path, subfolders, source_files )) + input("Read the above and press [Enter] to proceed...") # run duneproject - subprocess.call([dune_project]) + try: + logging.info("Calling dune-common/bin/duneproject to create the new extracted module...") + subprocess.call([dune_project]) + logging.info("--The new module is created sucessfully.") + except Exception: + logging.error("Failed to generate new module with duneproject!") # find the created folder # as the one with the most recent modification time + logging.debug("Getting the name of the new module...") new_module_name = max( [d for d in os.listdir() if os.path.isdir(d)], key=os.path.getmtime ) + logging.debug(f"Name of the new module is detected as {new_module_name}.") # verify it's really a Dune module + logging.debug("Checking if the new module is dune module...") check_module(new_module_name) + logging.debug("The new module is checked as dune module.") print( f"Found new module {new_module_name}\n" "Copying source files..." ) + logging.info("Extracting required headers and copy useful files to the new module...") # get the base path of the new module + logging.debug("Specifying paths of new module...") new_module_path = new_module_name.join(module_path.rsplit(module_dir,1)) + logging.debug(f"The path for new module is {new_module_path}.") # copy the source tree # copy all base folders complete, then delete the unnecessary ones + logging.debug("Copying old directories cotaining source files into new module...") base_folders = list(set([ os.path.relpath(s, module_path).split(os.path.sep)[0] for s in source_files ])) @@ -329,8 +359,10 @@ if __name__ == "__main__": path_in_old_module = os.path.join(module_path, b) path_in_new_module = os.path.join(new_module_path, b) copy_tree(path_in_old_module, path_in_new_module) + logging.debug("Directoies containing source files are copied.") # add base folders in project-level CMakeLists.txt + logging.debug("Adding base folders to CMakeList files in new module...") with open(os.path.join(new_module_path, "CMakeLists.txt"), "r") as cml: section = 0 content = [] @@ -346,8 +378,10 @@ if __name__ == "__main__": with open(os.path.join(new_module_path, "CMakeLists.txt"), "w") as cml: for line in content: cml.write(line) + logging.debug("CMakelist at top level in new module is confiugred.") # go through source tree and remove unnecessary directories + logging.debug("Handling directories where no source is obtained...") new_source_files = [ s.replace(module_dir, new_module_name, 1) for s in source_files ] @@ -397,29 +431,41 @@ if __name__ == "__main__": # make os.walk know about removed folders dirs.remove(d) + logging.debug("The folders containg no sources are handled.") # search for all header (in parallel) + logging.debug("Searching for all header files...") with mp.Pool() as p: headers = itertools.chain.from_iterable(p.map( partial(search_headers, module_path=module_path, headers=[]), source_files )) + logging.debug("Head files are found.") # make unique + logging.debug("Making header files unqiue (removing duplicates)...") headers = list(set(headers)) + logging.debug("Duplicates are removed.") # copy headers to the new module + logging.debug("Copying headers to the new module...") for header in headers: header_dir = os.path.dirname(os.path.realpath(header)) path_in_new_module = header_dir.replace(module_dir, new_module_name, 1) os.makedirs(path_in_new_module, exist_ok=True) shutil.copy(header, path_in_new_module) + logging.debug("Headers are copied to the new module.") # copy .gitignore from dumux to the new module dumux_gitignore_file = "dumux/.gitignore" + logging.debug("Copying .git from dumux into new module...") shutil.copy(dumux_gitignore_file, new_module_path) + logging.debug(".gitignore is copied.") + logging.info("--Requried headers are extracted and useful files are copied into the new module succesfully.") # delete unnecessary directories to keep the extracted module clean + logging.info("Cleaning up the new module...") + logging.debug("Deleting dune/src in the extracted module...") if "dune" not in subfolders: shutil.rmtree(os.path.join(new_module_path, 'dune')) if "src" not in subfolders: @@ -433,16 +479,21 @@ if __name__ == "__main__": content = content.replace("add_subdirectory(src)\n", "") cml.write(content) cml.truncate() + logging.debug("Dune/src are deleted.") + logging.info("--The new module is cleaned up successfully.") # create README file + logging.info("Creating README file and generating an install script in new module...") os.remove(os.path.join(new_module_path, 'README')) readme_path = os.path.join(new_module_path, "README.md") with open(readme_path, "w") as readme_file: readme_file.write( info_readme_main(module_dir, subfolders, source_files) ) + logging.debug("README file in new module is created.") # ask user if to write version information into README.md + logging.debug("Ask user if to write version information into README file.") if query_yes_no("Write detailed version information" " (folder/branch/commits/dates) into README.md?\n"): print("Looking for the dune modules in path: " @@ -459,16 +510,21 @@ if __name__ == "__main__": "\n## Version Information\n\n" + versionTable(versions) ) + logging.debug("The version information is written into README file.") # if there is a remote repository available # we can directly push the source code and - # also create a "one-click" installation scrip + # also create a "one-click" installation script + logging.debug("Generating an install script...") + logging.debug("Ask user to choose python or bash to generate the install script.") language = python_or_bash() install_script_name = 'install_' + new_module_name + '.%s'%("sh" if language == "bash" else "py") try: makeInstallScript(new_module_path, ignoreUntracked=True, skipFolders=new_module_name, suppressHints=True, topFolderName=None, language=language) + logging.debug("The install script is generated.") except Exception: + logging.error(f"Failed to generate script {install_script_name} by calling external functino makeInstallScript!") sys.exit( f"Automatically generate install script {install_script_name} failed." "\nTo create the script, use the 'makeinstallscript.py' script in the same folder." @@ -478,28 +534,54 @@ if __name__ == "__main__": shutil.move(install_script_name, os.path.join(new_module_path, install_script_name)) print(info_make_install(new_module_name)) + logging.info("--README file and install script are generated succesfully.") run_from_mod = callFromPath(new_module_path)(runCommand) + logging.info("Commiting the new module and pushing to remote if url is provided...") if query_yes_no("Do you have an empty remote repository to push the code to (recommended)?"): + logging.debug("Trying to get the remote URL.") remoteurl = get_remote_url(new_module_path) + logging.debug(f"The remote URL proviede by the user is {remoteurl}") # append install information into readme + logging.debug("Writing README file of the new module...") with open(readme_path, "a") as readme_file: readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name, language)) + logging.debug("The README file is written.") + logging.debug("Initializing git repository...") run_from_mod('git init') + logging.debug("Git repository is initialized.") + logging.debug("Adding all the files into git staging area...") run_from_mod('git add .') + logging.debug("All files are added into staging area.") + logging.debug("Commiting all files in staging area as inital commit...") run_from_mod('git commit -m "Initial commit"') + logging.debug("All files in staging area are commited.") + logging.debug(f"Adding remote {remoteurl} to the origin...") run_from_mod('git remote add origin {}'.format(remoteurl)) + logging.debug(f"{remoteurl} added to git remote origin.") + logging.debug(f"Pushing all the changes into remote git repository {remoteurl}") run_from_mod('git push -u origin master') + logging.debug("Commits are pushed to git remote.") # output guidance for users to create install script manually else: remoteurl = "{$remoteurl$} (needs to be manuelly adapted later)" # append install information into readme + logging.debug("Writing README file of the new module...") with open(readme_path, "a") as readme_file: readme_file.write(info_readme_installation(remoteurl, install_script_name, new_module_name, language)) + logging.debug("The README file is written.") + logging.debug("Initializing git repository...") run_from_mod('git init') + logging.debug("Git repository is initialized.") + logging.debug("Adding all the files into git staging area...") run_from_mod('git add .') + logging.debug("All files are added into staging area.") + logging.debug("Commiting all files in staging area as inital commit...") run_from_mod('git commit -m "Initial commit"') + logging.debug("All files in staging area are commited.") print("\nPlease remember to replace placeholder $remoteurl$ in installation part\n" "of the README file with the correct URL of your remote git repository.") + logging.info("--Changes are commited (and pushed if remote is known) successfully.") + logging.info("Congratulations, everything is fine and landing is smooth!") -- GitLab From 2f59025ad995f50046d34492c349895008ab6fca Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Mon, 5 Jul 2021 14:25:55 +0200 Subject: [PATCH 41/47] [bin][extract] Move cleaning no_source_dir in Cleaning part, Only consdier dir without useful headers --- bin/extractmodule/extract_as_new_module.py | 109 +++++++++++---------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 66592b4433..9e4e8d5254 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -346,7 +346,7 @@ if __name__ == "__main__": logging.info("Extracting required headers and copy useful files to the new module...") # get the base path of the new module logging.debug("Specifying paths of new module...") - new_module_path = new_module_name.join(module_path.rsplit(module_dir,1)) + new_module_path = new_module_name.join(module_path.rsplit(module_dir, 1)) logging.debug(f"The path for new module is {new_module_path}.") # copy the source tree @@ -380,59 +380,6 @@ if __name__ == "__main__": cml.write(line) logging.debug("CMakelist at top level in new module is confiugred.") - # go through source tree and remove unnecessary directories - logging.debug("Handling directories where no source is obtained...") - new_source_files = [ - s.replace(module_dir, new_module_name, 1) for s in source_files - ] - for b in base_folders: - def matching_path(path, list_of_paths): - for p in list_of_paths: - if path in p: - return True - return False - - no_source_folder = [] - for path, dirs, files in os.walk(os.path.join(new_module_path, b)): - for d in dirs: - dir_path = os.path.join(path, d) - keep = matching_path(dir_path, new_source_files) - if not keep: - rel_dir_path = os.path.relpath(dir_path, new_module_path) - no_source_folder.append(rel_dir_path) - if len(no_source_folder) > 1: - no_source_lists = ('\n'.join(dir for dir in no_source_folder)) - yes_to_all = query_yes_no( - "No source files found in the following directories:\n{}\n" - "Copy them all to the new module?\n" - "(Otherwise you need to configure each folder maneully)" - .format(no_source_lists) - ) - if not yes_to_all or len(no_source_folder) < 2: - for rel_dir_path in no_source_folder: - answer_is_yes = query_yes_no( - f"{rel_dir_path} does not contain source files." - " Copy to new module?", - default="yes" - ) - if not answer_is_yes: - # remove copy of directory - shutil.rmtree(dir_path) - # remove entry from CMakeLists.txt - cml_path = os.path.join(module_path, "CMakeLists.txt") - with open(cml_path, "r+") as cml: - content = cml.read() - cml.seek(0) - content = content.replace( - "add_subdirectory({d})\n", "" - ) - cml.write(content) - cml.truncate() - - # make os.walk know about removed folders - dirs.remove(d) - logging.debug("The folders containg no sources are handled.") - # search for all header (in parallel) logging.debug("Searching for all header files...") with mp.Pool() as p: @@ -449,9 +396,11 @@ if __name__ == "__main__": # copy headers to the new module logging.debug("Copying headers to the new module...") + header_dirs = [] for header in headers: header_dir = os.path.dirname(os.path.realpath(header)) path_in_new_module = header_dir.replace(module_dir, new_module_name, 1) + header_dirs.append(path_in_new_module) os.makedirs(path_in_new_module, exist_ok=True) shutil.copy(header, path_in_new_module) logging.debug("Headers are copied to the new module.") @@ -465,6 +414,58 @@ if __name__ == "__main__": # delete unnecessary directories to keep the extracted module clean logging.info("Cleaning up the new module...") + # go through source tree and remove unnecessary directories + logging.debug("Handling directories where no source is obtained...") + new_source_files = [ + s.replace(module_dir, new_module_name, 1) for s in source_files + ] + for b in base_folders: + def matching_path(path, list_of_paths, header_dirs): + for p in list_of_paths: + if path in p or path in header_dirs: + return True + return False + + no_source_folder = [] + for path, dirs, files in os.walk(os.path.join(new_module_path, b)): + for d in dirs: + dir_path = os.path.join(path, d) + keep = matching_path(dir_path, new_source_files, header_dirs) + if not keep: + rel_dir_path = os.path.relpath(dir_path, new_module_path) + no_source_folder.append(rel_dir_path) + + no_source_lists = ('\n'.join(dir for dir in no_source_folder)) + yes_to_all = query_yes_no( + "Could not automatically determine if following directories contain data essential for the extracted applications:\n{0}.\n" + "Do you want to copy all of them in the new module {1}?\n" + "(By choosing no you need to consider each folder seperately)" + .format(*[no_source_lists, new_module_name]), + default="yes" + ) + + if not yes_to_all: + for rel_dir_path in no_source_folder: + answer_is_yes = query_yes_no( + f"Could not automatically determine if {rel_dir_path} contains data essential for the extracted applications.\n" + f"Do you want to copy the files in {rel_dir_path} to the extracted module {new_module_name}", + default="yes" + ) + if not answer_is_yes: + # remove copy of directory + shutil.rmtree(os.path.join(new_module_path, rel_dir_path)) + # remove entry from CMakeLists.txt + cml_path = os.path.join(module_path, "CMakeLists.txt") + with open(cml_path, "r+") as cml: + content = cml.read() + cml.seek(0) + content = content.replace( + "add_subdirectory({d})\n", "" + ) + cml.write(content) + cml.truncate() + logging.debug("The folders containg no sources are handled.") + logging.debug("Deleting dune/src in the extracted module...") if "dune" not in subfolders: shutil.rmtree(os.path.join(new_module_path, 'dune')) -- GitLab From 7a1ab1a73b4008a5885d4972e203853df408a05d Mon Sep 17 00:00:00 2001 From: Kilian Weishaupt Date: Wed, 7 Jul 2021 09:25:51 +0000 Subject: [PATCH 42/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/makeinstallscript.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 70449c76d9..08ccceb055 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -28,8 +28,7 @@ if sys.version_info[0] < 3: # ask user to speficy the language of the generated script def python_or_bash(): - inp = input('In python or bash do you want to generate' - 'the install script?[p/b] (p:"Python", b:"Bash")') + inp = input('Generate install script in Python (p) or Bash (b)?') if inp == "b" or inp == "sh" or inp == "bash" or inp == "shell": print("You choose bash as language for install script.") return "bash" -- GitLab From 2d654317b2d88c414df4b87479eb7b2e58e1ad87 Mon Sep 17 00:00:00 2001 From: Kilian Weishaupt Date: Wed, 7 Jul 2021 09:26:45 +0000 Subject: [PATCH 43/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/makeinstallscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 08ccceb055..3ce9ce94a5 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -33,7 +33,7 @@ def python_or_bash(): print("You choose bash as language for install script.") return "bash" elif inp == "p" or inp == "py" or inp == "python": - print("You choose python as language for install script") + print("Creating Python install script") return "python" else: print("--Error: Unknown type. You must choose between python or bash!") -- GitLab From e9abbccb1aedb400241ba9a1d75b2846d4822ee0 Mon Sep 17 00:00:00 2001 From: Kilian Weishaupt Date: Wed, 7 Jul 2021 09:27:17 +0000 Subject: [PATCH 44/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/makeinstallscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 3ce9ce94a5..5df3001648 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -173,7 +173,7 @@ if __name__ == '__main__': ################### # parse arguments parser = argparse.ArgumentParser( - description='This script generates an install script for your dune module,' + description='This script generates an install script for your dune module, ' 'taking into account non-published commits & local changes.\n' 'This expects that your module is a git repository and that a ' 'remote origin exists and has been set already.' -- GitLab From cba2fbba6396c4a9fbaa429b91f71234639c4697 Mon Sep 17 00:00:00 2001 From: Kilian Weishaupt Date: Wed, 7 Jul 2021 09:27:23 +0000 Subject: [PATCH 45/47] Apply 1 suggestion(s) to 1 file(s) --- bin/extractmodule/makeinstallscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 5df3001648..634d735e59 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -30,7 +30,7 @@ if sys.version_info[0] < 3: def python_or_bash(): inp = input('Generate install script in Python (p) or Bash (b)?') if inp == "b" or inp == "sh" or inp == "bash" or inp == "shell": - print("You choose bash as language for install script.") + print("Creating Bash install script.") return "bash" elif inp == "p" or inp == "py" or inp == "python": print("Creating Python install script") -- GitLab From 86ed747f16c004cc128415512e099dcb6022fb0b Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 7 Jul 2021 12:53:23 +0200 Subject: [PATCH 46/47] [bin][makeInstall] Fix varibales with patches path instead of name --- bin/extractmodule/makeinstallscript.py | 31 ++++++++++++++++---------- bin/extractmodule/util.py | 24 ++++++++++++-------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/bin/extractmodule/makeinstallscript.py b/bin/extractmodule/makeinstallscript.py index 634d735e59..19be6df1cc 100644 --- a/bin/extractmodule/makeinstallscript.py +++ b/bin/extractmodule/makeinstallscript.py @@ -119,27 +119,34 @@ def makeInstallScript(path, optsPath = os.path.abspath(os.path.join(os.getcwd(), optsFile)) optsRelPath = os.path.relpath(optsPath, modParentPath) + names = depNames + if modName not in names and modFolder not in skipFolders: + names.append(modName) + folders = depFolders - if modName in depNames and modName not in folders: - folders.append(modName) + if modFolder not in folders and modFolder not in skipFolders: + folders.append(modFolder) # specific Module required patch and the patch path - patchRelPath = [] - patchModule = [] + patchFileType = [] # uncommitted or unpublished changes + patchModule = [] # names of remote modules containing channges + patchFolder = [] # names of local folders containing changes for depModPath in patches.keys(): - depModName = getModuleInfo(depModPath, "Module") + depModName = getModuleInfo(os.path.join(modParentPath, depModPath), "Module") if 'unpublished' in patches[depModPath]: - patchModule.append(depModPath) - patchRelPath.append(os.path.relpath("{}/unpublished.patch".format(depModName), depModPath)) + patchModule.append(depModName) + patchFolder.append(depModPath) + patchFileType.append("unpublished.patch") if 'uncommitted' in patches[depModPath]: - patchRelPath.append(os.path.relpath("{}/uncommitted.patch".format(depModName), depModPath)) - patchModule.append(depModPath) + patchFileType.append("uncommitted.patch") + patchModule.append(depModName) + patchFolder.append(depModPath) argsGenerateScript = (instFileName, - modName, modFolder, folders, versions, - patches, patchModule, patchRelPath, + modName, modFolder, + folders, names, versions, + patches, patchModule, patchFolder, patchFileType, topFolderName, optsRelPath) - if language == "bash": writeShellInstallScript(*argsGenerateScript) else: diff --git a/bin/extractmodule/util.py b/bin/extractmodule/util.py index 1e9c9a4768..0312a2f0d2 100644 --- a/bin/extractmodule/util.py +++ b/bin/extractmodule/util.py @@ -139,8 +139,8 @@ def printVersionTable(versions): def writeShellInstallScript(instFileName, modName, modFolder, - folders, versions, - patches, patchModule, patchRelPath, + folders, names, versions, + patches, patchModule, patchFolder, patchRelPath, topFolderName, optsRelPath="dumux/cmake.opts"): """ function to write the content into the generated shell install script @@ -154,11 +154,14 @@ def writeShellInstallScript(instFileName, - folders -- The list containing the folders of the module to be installed and all its dependencies + - names -- The list containing the names of the module to be + installed and all its dependencies - versions -- The persistent, remotely available git versions for the module to be installed and all its dependencies - patches -- The patches for unpublished commits and uncommitted changes - - patchModule -- The paths for the modules which has unpublished commits and uncommited changes + - patchModule -- The names for the modules which has unpublished commits and uncommited changes + - patchFolder -- The folders for the modules which has unpublished commits and uncommited changes - patchRelPath -- The realative paths of the generated patch files - topFolderName -- The of the folder that the install script creates upon execution to install the module in. @@ -225,7 +228,7 @@ def writeShellInstallScript(instFileName, installFile.write(')\n\n') installFile.write('DEPFOLDERS=(\n') - installFile.write('\n'.join(folders)) + installFile.write('\n'.join(names)) installFile.write('\n)\n\n') installFile.write('DEPBRANCHES=(\n') @@ -322,8 +325,8 @@ def writeShellInstallScript(instFileName, def writePythonInstallScript(instFileName, modName, modFolder, - folders, versions, - patches, patchModule, patchRelPath, + folders, names, versions, + patches, patchModule, patchFolder, patchRelPath, topFolderName, optsRelPath="dumux/cmake.opts"): """ function to write the content into the generated python install script @@ -337,11 +340,14 @@ def writePythonInstallScript(instFileName, - folders -- The list containing the folders of the module to be installed and all its dependencies + - names -- The list containing the names of the module to be + installed and all its dependencies - versions -- The persistent, remotely available git versions for the module to be installed and all its dependencies - patches -- The patches for unpublished commits and uncommitted changes - - patchModule -- The paths for the modules which has unpublished commits and uncommited changes + - patchModule -- The names for the modules which has unpublished commits and uncommited changes + - patchFolder -- The folders for the modules which has unpublished commits and uncommited changes - patchRelPath -- The realative paths of the generated patch files - topFolderName -- The of the folder that the install script creates upon execution to install the module in. @@ -436,8 +442,8 @@ if __name__ == '__main__': installFile.write(indent + r'deps = {}' + '\n') installFile.write(indent + 'deps["folders"] = [\n') - for dep in folders: - installFile.write(indent*2 + "\"" + dep + "\"," + '\n') + for name in names: + installFile.write(indent*2 + "\"" + name + "\"," + '\n') installFile.write(indent*2 + ']\n') installFile.write(indent + 'deps["branches"] = [\n') -- GitLab From 6a338cb44a5dbfb18b2652b36b3f2bafcda10233 Mon Sep 17 00:00:00 2001 From: Hanchuan Wu Date: Wed, 7 Jul 2021 21:56:50 +0200 Subject: [PATCH 47/47] [bin][extractModule] Support nested path and check if subdir --- bin/extractmodule/extract_as_new_module.py | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bin/extractmodule/extract_as_new_module.py b/bin/extractmodule/extract_as_new_module.py index 9e4e8d5254..de3f96a261 100755 --- a/bin/extractmodule/extract_as_new_module.py +++ b/bin/extractmodule/extract_as_new_module.py @@ -14,6 +14,7 @@ import re import multiprocessing as mp import fnmatch import itertools +from pathlib import Path from functools import partial from util import getPersistentVersions from util import versionTable @@ -112,12 +113,13 @@ def path_check(basedir, subdir): f"one level above the folder {basedir}.\n" f"Run \"{os.path.basename(__file__)} --help\" for details.") - # check if subdir immediate subfolder of basedir - list_subdir = [subdir.path for subdir in os.scandir(basedir) if subdir.is_dir()] - for folder in subdir: - if os.path.join(basedir, folder.strip(os.sep)) not in list_subdir: - logging.error(f"Subfolder '{folder}' is not a subfolder of '{basedir}'") - raise NameError(f"Subfolder '{folder}' is not a subfolder of '{basedir}'") + # check whether subdir is a sub-directory of basedir + base = Path(basedir) + for subfolder in subdir: + child = Path(os.path.join(base, subfolder)) + if base not in child.parents or not os.path.isdir(child): + logging.error(f"Subfolder '{subfolder}' is not a subfolder of '{basedir}'") + raise NameError(f"Subfolder '{subfolder}' is not a subfolder of '{basedir}'") def extract_sources_files(module_dir, subfolders): @@ -352,10 +354,7 @@ if __name__ == "__main__": # copy the source tree # copy all base folders complete, then delete the unnecessary ones logging.debug("Copying old directories cotaining source files into new module...") - base_folders = list(set([ - os.path.relpath(s, module_path).split(os.path.sep)[0] for s in source_files - ])) - for b in base_folders: + for b in subfolders: path_in_old_module = os.path.join(module_path, b) path_in_new_module = os.path.join(new_module_path, b) copy_tree(path_in_old_module, path_in_new_module) @@ -371,7 +370,7 @@ if __name__ == "__main__": section = 1 if section in [0, 1] else 2 elif section == 1: section = 2 - for b in base_folders: + for b in subfolders: content.append(f"add_subdirectory({b})") content.append(line) @@ -419,7 +418,7 @@ if __name__ == "__main__": new_source_files = [ s.replace(module_dir, new_module_name, 1) for s in source_files ] - for b in base_folders: + for b in subfolders: def matching_path(path, list_of_paths, header_dirs): for p in list_of_paths: if path in p or path in header_dirs: -- GitLab