Commit 35a6cba9 authored by Timo Koch's avatar Timo Koch
Browse files

Merge branch 'feature/refactor-install-script-generation-edit' into 'master'

feature/add-extract-module-python-script

Closes #1037 and #1021

See merge request !2709

(cherry picked from commit 559fcc21)

7ea7a7c1 [bin][util][common] Make util a package and write basic functions into common.py
ceaa8d30 [bin][util][moduleinfo] Rename script of dune-module information
7b653245 [bin][util]->[bin] Reconstrcut folder (move scripts to bin) and adapt imports
8d8ec200 [bin][util][writer] Add script to write install scirpt in different languages
3c4dd414 [bin][util][installscript] Add script to write helper function to make install script
7fe619e6 [bin] Rename, move and improve make_installscript, which is used to make an install script
993abbcc [bin][extractModule] Delete shell and provide new extract module script
parent 5e6a3b9f
Pipeline #6778 waiting for manual action with stages
#!/usr/bin/env python3
"""
Create files CMakeLists.txt for the given folder and all subfolders,
including the add_subdirectory(...) and install(...) commands.
Defaults to the folder `dumux` that contains the header files,
if no folder was specified.
"""
import os
import argparse
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('folder', type=str, nargs='?', help='the folder to create CMakeLists.txt\'s for', default=None)
args = vars(parser.parse_args())
# default to the dumux folder (relative path to the location of this script)
if args['folder'] is None:
rootDir = os.path.dirname(os.path.abspath(__file__)) + "/../../dumux"
else:
rootDir = args['folder']
ignore_folders = ["", "io/format/fmt", "io/xml"]
extensions = [".hh", ".inc"]
for fullFolderName, subFolders, files in os.walk(rootDir):
# alphabetically sort
subFolders = sorted(subFolders)
files = sorted(files)
# get folder name relative to dumux
folderName = fullFolderName.replace(rootDir + '/', '').replace(rootDir, '')
if folderName not in ignore_folders:
with open(fullFolderName + "/CMakeLists.txt", "w") as cmakelists:
# add subfolders
for subFolder in subFolders:
cmakelists.write("add_subdirectory({})\n".format(subFolder))
headersExist = False
for fileName in files:
ext = os.path.splitext(fileName)[1]
if ext in extensions:
headersExist = True
break
if headersExist:
if subFolders: cmakelists.write("\n")
# collect all files to be installed in a CMake variable
headers_variable = "DUMUX_" + folderName.upper().replace("/", "_") + "_HEADERS"
cmakelists.write("file(GLOB {}{})\n".format(headers_variable, " *".join([''] + extensions)))
cmakelists.write("install(FILES ${{{}}}\n".format(headers_variable))
cmakelists.write(" DESTINATION ${{CMAKE_INSTALL_INCLUDEDIR}}/dumux/{})\n".format(folderName))
#!/usr/bin/env python3
import os
import sys
import string
import shutil
import argparse
import subprocess
from util.moduleinfo import getModuleFile
from util.moduleinfo import extractModuleInfos
# require python 3
if sys.version_info[0] < 3:
sys.exit("\nERROR: Python3 required")
if __name__ == "__main__":
# input argument parser
parser = argparse.ArgumentParser(
description="Create a docker image for a given module and install script."
)
parser.add_argument('-m', '--modulepath',
required=True,
help='the path to the your module')
parser.add_argument('-i', '--installScript',
required=True,
help="Specify the installation script")
parser.add_argument('-t', '--templateFolder',
required=False,
help="Specify the folder with the template files")
args = vars(parser.parse_args())
# get information on the module
modulePath = os.path.abspath(args['modulepath'])
modInfo = extractModuleInfos(getModuleFile(modulePath),
['Module', 'Maintainer'])
moduleName = modInfo['Module']
moduleMaintainer = modInfo['Maintainer']
dockerTag = moduleName.lower() # docker only supports lower case
# get folder with the template files
templateFolder = args['templateFolder']
if not templateFolder:
templateFolder = os.path.join(modulePath, '../dumux/docker')
if not os.path.exists(templateFolder):
sys.exit("Template folder {} could not be found".format(templateFolder))
print("*"*54)
print("\n-- Creating a Docker image for module " + moduleName + " --\n")
print("*"*54)
if os.path.exists("docker"):
print("\nA docker folder already exists. "
"Continue anyway? - will be overwritten - [y/N]\n")
delete = input()
if delete == "y" or delete == "Y":
shutil.rmtree("docker")
print("--> Deleted old docker folder.")
else:
sys.exit("Abort.")
os.mkdir("docker")
print("--> Created the folder 'docker'.")
# copy install script into docker folder and make it executable
installScriptPath = args['installScript']
installScriptName = os.path.split(installScriptPath)[1]
installScript = os.path.join(os.path.join(os.getcwd(), 'docker'),
installScriptName)
shutil.copy(installScriptPath, installScript)
os.system("chmod +x {}".format(installScript))
print("--> Using install script: {} to install dependencies for module {}."
.format(installScript, moduleName))
# substitute content from template and write to target
def substituteAndWrite(template, target, mapping):
if not os.path.exists(template):
sys.exit("Template file '" + template + "' could not be found")
with open(target, 'w') as targetFile:
raw = string.Template(open(template).read())
targetFile.write(raw.substitute(**mapping))
# write setpermissions helper script
template = os.path.join(templateFolder, 'setpermissions.sh.template')
target = os.path.join(os.getcwd(), 'docker/setpermissions.sh')
substituteAndWrite(template, target, {})
print("--> Created permission helper script for easier container setup.")
# write welcome message file
template = os.path.join(templateFolder, 'WELCOME.template')
target = os.path.join(os.getcwd(), 'docker/WELCOME')
substituteAndWrite(template, target,
{'modName': moduleName, 'modFolder': moduleName})
print("--> Created welcome message displayed on Docker container startup.")
# write readme file
template = os.path.join(templateFolder, 'README.md.template')
target = os.path.join(os.getcwd(), 'docker/README.md')
substituteAndWrite(template, target,
{'modName': moduleName, 'dockerTag': dockerTag})
print("--> Created README.md on how to use the docker image.")
# write helper file for container spin-up (make it executable after creation)
template = os.path.join(templateFolder, 'docker.sh.template')
target = os.path.join(os.getcwd(), 'docker/docker_{}.sh'.format(dockerTag))
substituteAndWrite(template, target, {'dockerTag': dockerTag})
os.system("chmod +x " + target)
print("--> Created helper script to spin up the docker container.")
# write the docker file
template = os.path.join(templateFolder, 'Dockerfile.template')
target = os.path.join(os.getcwd(), 'docker/Dockerfile')
substituteAndWrite(template, target,
{
'modName': moduleName,
'modMaintainer': moduleMaintainer,
'dockerTag': dockerTag,
'instScript': installScriptName
})
print("--> Created Dockerfile. You can adapt it to your needs.")
print()
print("Do you want to directly build the Docker image? [y/N]")
build = input()
if build == "y" or build == "Y":
print("Building Docker image... this may take several minutes.")
try:
os.chdir('docker')
subprocess.run(['docker', 'build',
'-f', 'Dockerfile',
'-t', dockerTag, '.'], check=True)
os.chdir('../')
except Exception:
os.chdir('../')
sys.exit("ERROR: docker image build failed")
print()
print("Successfully built image: {}. "
"Have a look at docker/README.md.".format(dockerTag))
print("Check the container by running "
"'docker run -it {} /bin/bash' in the same".format(dockerTag))
print("directory as the Dockerfile, and try using the convenience script "
"docker_{}.sh".format(dockerTag))
print("See docker/README.md for more information.")
else:
print("You can build your Docker image later by running "
"'docker build -f Dockerfile -t {}'".format(dockerTag))
print("from within the folder 'docker' that was created by this script, "
"and in which you should find the 'Dockerfile'.")
This diff is collapsed.
#!/usr/bin/env python3
""""
Script to generate an install script for a dune-module,
accounting for non-published commits and local changes
"""
import sys
import os
import argparse
import subprocess
from util.moduleinfo import getDependencies, getModuleInfo
from util.installscript import (
addDependencyPatches,
addDependencyVersions,
getDefaultScriptName,
filterDependencies,
makeInstallScript,
makeScriptWriter,
printProgressInfo,
printFoundDependencies,
printFoundVersionInfo,
printFinalMessage,
supportedLanguages,
)
if __name__ == "__main__":
###################
# parse arguments
parser = argparse.ArgumentParser(
description="This script generates an install script for your module, "
"taking into account non-published commits & changes.\n"
"This expects that all modules are git repositories and "
"have a remote origin URL defined."
)
parser.add_argument(
"-p", "--path", required=True, help="The path to your dune module"
)
parser.add_argument(
"-f",
"--filename",
required=False,
help="File in which to write the install script",
)
parser.add_argument(
"-i",
"--ignoreuntracked",
required=False,
action="store_true",
help="Use this to ignore untracked files present",
)
parser.add_argument(
"-t",
"--topfoldername",
required=False,
default="DUMUX",
help="Folder that the install script creates upon"
"execution to install the modules in. If you "
"pass an empty string, no folder will be created "
"and installation happens in place.",
)
parser.add_argument(
"-o",
"--optsfile",
required=False,
help="Provide custom opts file to be used for project "
"configuration. Note that this file is required "
"to be contained and committed in the module or "
"one of its dependencies.",
)
parser.add_argument(
"-s",
"--skipfolders",
required=False,
nargs="+",
help="a list of module folders to be skipped",
)
parser.add_argument(
"-l",
"--language",
required=False,
default="python",
choices=supportedLanguages(),
help="Language in which to write the install script",
)
cmdArgs = vars(parser.parse_args())
modPath = cmdArgs["path"]
skipFolders = cmdArgs["skipfolders"]
if skipFolders:
skipFolders = list(set(skipFolders))
printProgressInfo(["Determining the module dependencies"])
deps = getDependencies(modPath, verbose=True, includeSelf=True)
deps = filterDependencies(deps, skipFolders)
printFoundDependencies(deps)
if not deps:
sys.exit("No dependencies found. Exiting.")
printProgressInfo(["Determining the module versions"])
deps = addDependencyVersions(deps, cmdArgs.get("ignoreuntracked", False))
printFoundVersionInfo(deps)
printProgressInfo(["Making patches for unpublished & uncommited changes"])
deps = addDependencyPatches(deps)
# actual script generation
modPath = os.path.abspath(modPath)
modName = getModuleInfo(modPath, "Module")
printProgressInfo(
[
"Creating install script for module '{}' in folder '{}'".format(
modName, modPath
)
]
)
language = cmdArgs["language"]
scriptName = cmdArgs.get("filename", None)
if not scriptName:
scriptName = getDefaultScriptName(modName, language)
makeInstallScript(
modPath=modPath,
dependencies=deps,
scriptName=scriptName,
writer=makeScriptWriter(language),
topFolderName=cmdArgs.get("topfoldername", None),
optsFile=cmdArgs.get("optsFile", None),
)
subprocess.call(["chmod", "u+x", scriptName])
printProgressInfo([f"Successfully created install script '{scriptName}'"])
printFinalMessage(scriptName, cmdArgs.get("topfoldername", None))
import os
import re
import sys
import fnmatch
import functools
import subprocess
import traceback
import datetime
import shlex
TERM_FORMATTING = {
"warning": "\033[1;31m",
"highlight": "\033[1;34m",
"reset": "\033[0m",
"none": "",
}
def styledBotPrint(s, style="none", **kwargs):
sys.stdout.write("\n🤖 ")
sys.stdout.write(TERM_FORMATTING[style])
print(s, **kwargs)
sys.stdout.write(TERM_FORMATTING["reset"])
def addPrefix(prefix, text, separator=' '):
return prefix + separator + text
def addPrefixToLines(prefix, text, separator=' '):
return '\n'.join(
addPrefix(prefix, line, separator) for line in text.split('\n')
)
def escapeCharacter(text, character, escCharacter="\\"):
return text.replace(character, f'{escCharacter}{character}')
def escapeCharacters(text, characters, escCharacter="\\"):
for char in characters:
text = escapeCharacter(text, char, escCharacter)
return text
def indent(text, indentation=' '):
text = text.split('\n')
text = [indentation + line for line in text]
return '\n'.join(text)
def makeTable(dictList, config=None, padding=2):
if config is None:
config = {key: key for d in dictList for key in d}
def getColWidth(row):
return max(len(str(r)) for r in row) + padding*2
def getCol(key):
return [config[key]] + [d.get(key, "") for d in dictList]
widths = {key: getColWidth(getCol(key)) for key in config}
def makeRow(rowValues):
row = "|"
for key in config:
row += "{}|".format(rowValues.get(key, "").center(widths[key]))
return row
table = [makeRow({key: config[key] for key in config})]
table.append('|' + '|'.join('-'*widths[key] for key in config) + '|')
table.extend(makeRow(row) for row in dictList)
return '\n'.join(table)
def getCommandErrorHints(command):
......@@ -13,12 +82,13 @@ def getCommandErrorHints(command):
return None
# execute a command and retrieve the output
def runCommand(command, suppressTraceBack=False, errorMessage=''):
def runCommand(command, check=True, suppressTraceBack=False, errorMessage=''):
"""execute a command and retrieve the output"""
try:
return subprocess.run(command,
shell=True, check=True,
text=True, capture_output=True).stdout
return subprocess.run(
shlex.split(command), check=check, text=True, capture_output=True
).stdout
except Exception:
eType, eValue, eTraceback = sys.exc_info()
if suppressTraceBack:
......@@ -35,8 +105,9 @@ def runCommand(command, suppressTraceBack=False, errorMessage=''):
print(hints)
# decorator to call function from within the given path
def callFromPath(path):
"""decorator to call function from within the given path"""
def decorator_callFromPath(callFunc):
@functools.wraps(callFunc)
def wrapper_callFromPath(*args, **kwargs):
......@@ -47,3 +118,245 @@ def callFromPath(path):
return result
return wrapper_callFromPath
return decorator_callFromPath
def userQuery(query, choices=None):
"""query something from the user"""
choicesString = ', '.join(str(c) for c in choices) if choices else ''
querySuffix = f" (choices: {choicesString})\n" if choices else ' '
while True:
styledBotPrint(f"{query.strip()}{querySuffix}", style="highlight")
inp = input()
if choices and inp not in choices:
styledBotPrint(
f"Invalid answer: '{inp}'. Choose from {choicesString}.",
style="warning"
)
else:
return inp
def queryYesNo(question, default="yes"):
"""query a yes/no answer from the user"""
affirmative = ["yes", "y", "ye"]
negative = ["no", "n"]
def getChoices():
return ", ".join(c for c in affirmative + negative)
def isAffirmative(choice): return choice in affirmative
def isNegative(choice): return choice in negative
def isValid(choice): return isAffirmative(choice) or isNegative(choice)
if default is not None and not isValid(default):
raise ValueError("\nInvalid default answer: '{}', choices: '{}'\n"
.format(default, getChoices()))
if default is None:
prompt = " [y/n] "
else:
prompt = " [Y/n] " if isAffirmative(default) else " [y/N] "
while True:
styledBotPrint(f"{question.strip()}{prompt}", style="highlight", end="")
choice = input().lower()
if default is not None and choice == "":
return True if isAffirmative(default) else False
if not isValid(choice):
styledBotPrint(
f"Invalid answer: '{choice}'. Choose from '{getChoices()}'",
style="warning"
)
else:
return True if isAffirmative(choice) else False
def cppHeaderFilter():
return lambda fileName: fileName == 'config.h'
def includedCppProjectHeaders(file,
projectBase,
headers=[],
headerFilter=cppHeaderFilter()):
"""get all project headers included by a cpp file"""
filePath = os.path.join(projectBase, file)
if not os.path.exists(filePath):
raise IOError(f'Cpp file {filePath} does not exist')
with open(filePath, 'r') as f:
content = f.read()
headerInBracket = re.findall(r'#include\s+<(.+?)>', content)
headerInQuotation = re.findall(r'#include\s+"(.+?)"', content)
def process(pathInProject):
headerPath = os.path.join(projectBase, pathInProject)
if os.path.exists(headerPath):
if not headerFilter(pathInProject):
if headerPath not in headers:
headers.append(headerPath)
includedCppProjectHeaders(
headerPath, projectBase,
headers, headerFilter
)
for header in headerInBracket:
process(header)
for header in headerInQuotation:
absHeaderPath = os.path.join(os.path.dirname(file), header)
projectPath = os.path.relpath(absHeaderPath, projectBase)
process(projectPath)
return headers
def findMatchingFiles(path, pattern):
"""find all files below the given folder that match the given pattern"""
result = []
for root, dirs, files in os.walk(path):
relativeRootPath = os.path.relpath(root, path)
for file in files:
if fnmatch.fnmatch(file, pattern):
result.append(os.path.join(relativeRootPath, file))
return result
def isGitRepository(pathToRepo='.'):
try:
run = callFromPath(pathToRepo)(runCommand)
run('git status')
return True
except Exception:
return False
def getRemote(pathToRepo='.'):
run = callFromPath(pathToRepo)(runCommand)
return run('git ls-remote --get-url').strip('\n')
def fetchRepo(remote, pathToRepo='.'):
run = callFromPath(pathToRepo)(runCommand)
run('git fetch {}'.format(remote))
def hasUntrackedFiles(pathToRepo='.'):
run = callFromPath(pathToRepo)(runCommand)
return run('git ls-files --others --exclude-standard') != ''
def isPersistentBranch(branchName):
if branchName == 'origin/master':
return True
if branchName.startswith('origin/releases/'):
return True
return False