Commit fffd17dd authored by Hanchuan Wu's avatar Hanchuan Wu
Browse files

Merge branch 'feature/shell2python/installexternal' into 'master'

Rewrite Install external with Python

Closes #866

See merge request !2252
parents a73b02de b43b9e0e
......@@ -10,6 +10,7 @@ Differences Between DuMu<sup>x</sup> 3.4 and DuMu<sup>x</sup> 3.3
- Advection type including inertia effect for simulations in the non-creeping flow regime
- Note that this is still considered a rather _experimental_ feature. Everything within namespace `Dumux::PoreNetwork` might undergo (backward-compatibility breaking) changes _without prior notice_.
- __Several scripts have been translated to Python__:
- `installexternal.sh` to install external modules and libraries now rewritten as python script `bin/installexternal.py`
- `getusedversions.sh` to extract the used Dumux/Dune versions of a module (new script: `bin/util/getusedversions.py`)
- `extractmodulepart.sh` no longer creates an install file, instead, you can now generate install scripts for your module using the new script `bin/util/makeinstallscript.py`.
- Note: the old shells script will be removed after release 3.4.
......
#!/usr/bin/env python3
"""
install external stuff for dumux
"""
import os
import urllib.request
import tarfile
import sys
import subprocess
import shutil
import re
import argparse
class ChoicesAction(argparse._StoreAction):
def __init__(self, **kwargs):
super(ChoicesAction, self).__init__(**kwargs)
if self.choices is None:
self.choices = []
self._choices_actions = []
def add_choice(self, choice, help=''):
self.choices.append(choice)
choice_action = argparse.Action(option_strings=[], dest=choice, help=help)
self._choices_actions.append(choice_action)
def _get_subactions(self):
return self._choices_actions
def show_message(message):
print("*" * 120)
print(message)
print("")
print("*" * 120)
if len(sys.argv) == 1:
show_message('No options given. For more information run the following command: \n ./installexternal.py --help')
sys.exit()
parser = argparse.ArgumentParser(prog='installexternal',
usage='./installexternal.py [OPTIONS] PACKAGES',
description='This script downloads extenstions for dumux and dune \
and installs some External Libraries and Modules.')
parser.register('action', 'store_choice', ChoicesAction)
# Positional arguments
group = parser.add_argument_group(title='your choice of packages')
packages = group.add_argument('packages', nargs='+', metavar='PACKAGES',
action='store_choice')
packages.add_choice('dumux-extensions', help="Download dumux-course and dumux-lecture.")
packages.add_choice('dune-extensions', help="Download dune-uggrid, dune-alugrid, dune-foamgrid, \
dune-subgrid, dune-spgrid, dune-mmesh and dune-functions.")
packages.add_choice('optimization', help="Download and install glpk and nlopt.")
packages.add_choice('others', help="Download and install opm , metis and gstat.")
packages.add_choice('lecture', help="Download dumux-lecture.")
packages.add_choice('course', help="Download dumux-course.")
packages.add_choice('ug', help="Download dune-uggrid.")
packages.add_choice('alugrid', help="Download dune-alugrid.")
packages.add_choice('foamgrid', help="Download dune-foamgrid.")
packages.add_choice('subgrid', help="Download dune-subgrid.")
packages.add_choice('spgrid', help="Download dune-spgrid.")
packages.add_choice('mmesh', help="Download dune-mmesh.")
packages.add_choice('functions', help="Download dune-functions.")
packages.add_choice('glpk', help="Download and install glpk.")
packages.add_choice('nlopt', help="Download and install nlopt.")
packages.add_choice('opm', help="Download opm modules required for cornerpoint grids.")
packages.add_choice('metis', help="Download and install the METIS graph partitioner.")
packages.add_choice('gstat', help="Download and install gstat.")
# Optional arguments
options = parser.add_mutually_exclusive_group(required=False)
options.add_argument('--clean', action="store_true", default=False,
help='Delete all files for the given packages.')
options.add_argument('--download', action="store_true", default=False,
help='Only download the packages.')
parser.add_argument('--dune_branch', default="releases/2.7",
help='Dune branch to be checked out.')
parser.add_argument('--dumux_branch', default="releases/3.3",
help='Dumux branch to be checked out.')
parser.add_argument('--opm_branch', default="release/2020.10",
help='Opm branch to be checked out.')
parser.add_argument('--mmesh_branch', default="release/1.2",
help='Mmesh branch to be checked out.')
args = vars(parser.parse_args())
def run_command(command, currentdir='.'):
with open(currentdir+"/installexternal.log", "a") as log:
popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
for line in popen.stdout:
log.write(line)
print(line, end='')
for line in popen.stderr:
log.write(line)
print(line, end='')
popen.stdout.close()
popen.stderr.close()
return_code = popen.wait()
if return_code:
print("\n")
message = "\n (Error) The command {} returned with non-zero exit code\n".format(command)
message += "\n If you can't fix the problem yourself consider reporting your issue\n"
message += " on the mailing list (dumux@listserv.uni-stuttgart.de) and attach the file 'installexternal.log'\n"
show_message(message)
sys.exit(1)
def git_clone(url, branch=None):
clone = ["git", "clone"]
if branch:
clone += ["-b", branch]
result = run_command(command=[*clone, url])
def install_external(args):
dune_branch = args['dune_branch']
dumux_branch = args['dumux_branch']
opm_branch = args['opm_branch']
mmesh_branch = args['mmesh_branch']
packages = args['packages']
cleanup = args['clean']
download = args['download']
final_message = []
top_dir = os.getcwd()
ext_dir = top_dir + "/external"
# Prepare a list of packages
packages = []
for pkg in args['packages']:
if pkg in packagenames:
packages.extend(packagenames[pkg])
else:
packages.extend([key for key in external_urls.keys() if pkg in key])
args['packages'] = packages
# print the list of packages to be downloaded/installed/removed
print("The following package(s) will be {0}:\n".format('removed' if cleanup else 'downloaded'), ', '.join(args['packages']), "\n")
# check Location For DuneModules
if not os.path.isdir("dune-common"):
show_message(
"You have to call " + sys.argv[0] + " for " + sys.argv[1] + " from\n"
"the same directory in which dune-common is located.\n"
"You cannot install it in this folder.")
return
# clear the log file
logdir = ext_dir if os.path.exists(ext_dir) else top_dir
open(logdir+'/installexternal.log', 'w').close()
for package in packages:
os.chdir(top_dir)
# Package name for final message
final_message.append('[---'+package+'---]')
# Set the directory: create ext_dir for external packages
if not any([re.compile(p).match(package) for p in ['dumux','dune', 'opm']]):
os.makedirs(ext_dir, exist_ok=True)
os.chdir(ext_dir)
# Set the branch
if 'dumux' in package:
branch = dumux_branch
elif 'mmesh' in package:
branch = mmesh_branch
elif 'dune' in package:
branch = dune_branch
elif 'opm' in package:
branch = opm_branch
# Run the requested command
if cleanup:
if os.path.isfile(package + '.tar.gz'):
os.remove(package + '.tar.gz')
if os.path.exists(package):
# Remove
shutil.rmtree(package)
# Save message to be shown at the end
final_message.append("{} has been removed.".format(package))
else:
# Save message to be shown at the end
final_message.append("The folder {} does not exist.".format(package))
continue
else:
# Check if tarball
tarball = external_urls[package].endswith('tar.gz')
if not os.path.exists(package):
if tarball:
# Download the tarfile
filedata = urllib.request.urlopen(external_urls[package])
datatowrite = filedata.read()
with open(ext_dir + "/" + package +".tar.gz", 'wb') as f:
f.write(datatowrite)
# Save message to be shown at the end
final_message.append("{} has been sucessfully downloaded.".format(package))
# Start Installation if the flag download is set to false.
if not download:
# Extract
tf = tarfile.open(package+".tar.gz")
tf.extractall()
# Rename
shutil.move(os.path.commonprefix(tf.getnames()), package)
# Start the configuration
os.chdir(ext_dir + "/"+package)
if package == 'gstat':
with open('configure', 'r+') as f:
content = f.read()
f.seek(0)
f.truncate()
f.write(content.replace('doc/tex/makefile', ''))
# Run Configuration command
configcmd = "./configure" if package != 'metis' else ["make", "config"]
run_command(configcmd, currentdir=ext_dir)
try:
run_command("make", currentdir=ext_dir)
except:
raise Exception("{} installation has failed.".format(package))
# Save message to be shown at the end
if os.path.exists(ext_dir+ "/" + package):
final_message.append("{} has been successfully installed.".format(package))
else:
# Clone from repo
git_clone(external_urls[package], branch)
# Save message to be shown at the end
final_message.append("{} has been sucessfully cloned.".format(package))
else:
if tarball:
final_message.append("{} has been already installed.".format(package))
else:
# Checkout to the requested branch
os.chdir(top_dir + '/' + package)
subprocess.Popen(["git", "checkout", branch])
# Save message to be shown at the end
final_message.append("-- Skip cloning {}, because the folder already exists.".format(package))
final_message.append("-- Checking out {} ".format(package) + branch)
continue
# Save post installation message if there is any.
if package in messages.keys():
final_message.extend(messages[package])
# Change to top_dir
os.chdir(top_dir)
# Save post installation message about dunecontrol if need be.
if not cleanup and any(x in pkg for pkg in packages for x in ["dumux","dune","opm"]):
final_message.append("\n\nPlease run the following command (can be copied to command line):\n\n ./dune-common/bin/dunecontrol --opts=./dumux/cmake.opts all")
# If cleanup and only logfile in the external directory, remove the directory
if os.path.isdir(ext_dir):
_, _, files = next(os.walk(ext_dir))
if cleanup and len(files)==1 and 'installexternal.log' in files:
shutil.rmtree(ext_dir)
return '\n'.join(final_message)
#################################################################
#################################################################
## (1/3) Define th necessary packages and their urls
#################################################################
#################################################################
dune_git_baseurl = "https://gitlab.dune-project.org/"
dumux_git_baseurl = "https://git.iws.uni-stuttgart.de/dumux-repositories/"
external_urls = {
"dumux-lecture": dumux_git_baseurl + "dumux-lecture.git",
"dumux-course": dumux_git_baseurl + "dumux-course.git",
"dune-uggrid": dune_git_baseurl + "/staging/dune-uggrid.git",
"dune-alugrid": dune_git_baseurl + "extensions/dune-alugrid.git",
"dune-foamgrid": dune_git_baseurl + "extensions/dune-foamgrid.git",
"dune-subgrid": "https://git.imp.fu-berlin.de/agnumpde/dune-subgrid.git",
"dune-spgrid": dune_git_baseurl + "extensions/dune-spgrid.git",
"dune-mmesh": dune_git_baseurl + "samuel.burbulla/dune-mmesh.git",
"dune-functions": dune_git_baseurl + "staging/dune-functions.git",
"dune-typetree": dune_git_baseurl + "staging/dune-typetree.git",
"glpk": "http://ftp.gnu.org/gnu/glpk/glpk-4.60.tar.gz",
"nlopt": "http://ab-initio.mit.edu/nlopt/nlopt-2.4.2.tar.gz",
"opm-common": "https://github.com/OPM/opm-common",
"opm-grid": "https://github.com/OPM/opm-grid",
"metis": "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz",
"gstat": "http://gstat.org/gstat.tar.gz",
}
packagenames = {
"dumux-extensions": ["dumux-lecture", "dumux-course"],
"dune-extensions": ["dune-uggrid", "dune-alugrid", "dune-foamgrid",
"dune-subgrid", "dune-spgrid", "dune-mmesh",
"dune-functions", "dune-typetree"],
"functions": ["dune-functions", "dune-typetree"],
"optimization": ["glpk", "nlopt"],
"others": ["opm-common", "opm-grid", "metis", "gstat"]
}
messages ={
'glpk': ["In addition, it might be necessary to set manually",
"the glpk path in the CMAKE_FLAGS section of the .opts-file:",
" -DGLPK_ROOT=/path/to/glpk \\"],
'dune-mmesh': ["Maybe you also have to install CGAL",
"(see cgal.org/download.html)", "Maybe you also need to change your core dune modules' branches to their newest versions."],
'opm-common': ["In addition, it might be necessary to set manually some",
"CMake variables in the CMAKE_FLAGS section of the .opts-file:",
" -DUSE_MPI=ON",
"Currently, compiling opm with clang is not possible.", "",
"Maybe you also have to install the following packages (see the",
" opm prerequisites at opm-project.org):",
" BLAS, LAPACK, Boost, SuiteSparse, Zoltan"]
}
#################################################################
#################################################################
## (2/3) Download/Config/Clean the requested packages
#################################################################
#################################################################
# Start download/configuration/cleaning tasks
final_message = install_external(args)
#################################################################
#################################################################
## (3/3) Show the final message
#################################################################
#################################################################
# Show final message
show_message(final_message)
#!/bin/bash
CORRECT_LOCATION_FOR_DUNE_MODULES="n"
ENABLE_MPI="n"
ENABLE_DEBUG="n"
ENABLE_PARALLEL="n"
CLEANUP="n"
DOWNLOAD_ONLY="n"
TOPDIR=$(pwd)
EXTDIR=$(pwd)/external
DUMUX_VERSION=3.2
checkLocationForDuneModules()
{
# test for directory dune-common, dune-common-2.4 etc.
if ! ls dune-common* &> /dev/null; then
echo "You have to call $0 for $1 from"
echo "the same directory in which dune-common is located."
echo "You cannot install it in this folder."
CORRECT_LOCATION_FOR_DUNE_MODULES="n"
return
fi
CORRECT_LOCATION_FOR_DUNE_MODULES="y"
}
createExternalDirectory()
{
if [ ! -e $EXTDIR ]; then
mkdir -v $EXTDIR
fi
}
installAluGrid()
{
cd $TOPDIR
checkLocationForDuneModules dune-alugrid
if test $CORRECT_LOCATION_FOR_DUNE_MODULES == "n"; then
return
fi
if [ ! -e dune-alugrid ]; then
git clone -b releases/2.4 https://gitlab.dune-project.org/extensions/dune-alugrid.git
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf dune-alugrid
return
fi
}
installCourse()
{
cd $TOPDIR
if [ ! -d "dumux-course" ]; then
git clone -b releases/$DUMUX_VERSION https://git.iws.uni-stuttgart.de/dumux-repositories/dumux-course.git
else
echo "Skip cloning dumux-course because the folder already exists."
cd dumux-course
git checkout releases/$DUMUX_VERSION
cd ..
fi
}
installFoamGrid()
{
cd $TOPDIR
checkLocationForDuneModules dune-foamgrid
if test $CORRECT_LOCATION_FOR_DUNE_MODULES == "n"; then
return
fi
if [ ! -e dune-foamgrid ]; then
git clone -b releases/2.4 https://gitlab.dune-project.org/extensions/dune-foamgrid.git
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf dune-foamgrid
return
fi
}
installGLPK()
{
cd $EXTDIR
rm -rf glpk* standalone
if [ ! -e glpk-4.60.tar.gz ]; then
wget http://ftp.gnu.org/gnu/glpk/glpk-4.60.tar.gz
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf glpk
return
fi
mkdir glpk
tar zxvf glpk-4.60.tar.gz --strip-components=1 -C glpk
cd glpk
./configure
make
# show additional information
echo "In addition, it might be necessary to set manually"
echo "the glpk path in the CMAKE_FLAGS section of the .opts-file:"
echo " -DGLPK_ROOT=/path/to/glpk \\"
cd $TOPDIR
}
installGStat()
{
cd $EXTDIR
rm -rf gstat* standalone
if [ ! -e gstat.tar.gz ]; then
wget http://gstat.org/gstat.tar.gz
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
mkdir gstat
tar zxvf gstat.tar.gz --strip-components=1 -C gstat
cd gstat
sed -i 's# doc/tex/makefile##g' configure
./configure
make
if [ -e $PWD/src/gstat ]; then
echo "Successfully installed gstat."
fi
}
installLecture()
{
cd $TOPDIR
if [ ! -d "dumux-lecture" ]; then
git clone -b releases/$DUMUX_VERSION https://git.iws.uni-stuttgart.de/dumux-repositories/dumux-lecture.git
else
echo "Skip cloning dumux-lecture because the folder already exists."
cd dumux-lecture
git checkout releases/$DUMUX_VERSION
cd ..
fi
}
installMETIS()
{
cd $EXTDIR
if [ ! -e metis-5.1.0.tar.gz ]; then
wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf metis-5.1.0
return
fi
if ! test -e "metis-5.1.0"; then
tar zxvf metis-5.1.0.tar.gz
fi
cd metis-5.1.0
METISDIR=$(pwd)
make config
make
cd $TOPDIR
}
installMultidomain()
{
cd $TOPDIR
checkLocationForDuneModules dune-multidomain
if test $CORRECT_LOCATION_FOR_DUNE_MODULES == "n"; then
return
fi
if [ ! -e dune-multidomain ]; then
git clone -b releases/2.0 git://github.com/smuething/dune-multidomain.git
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf dune-multidomain
return
fi
cd $TOPDIR
}
installMultidomainGrid()
{
cd $TOPDIR
checkLocationForDuneModules dune-multidomaingrid
if test $CORRECT_LOCATION_FOR_DUNE_MODULES == "n"; then
return
fi
if [ ! -e dune-multidomaingrid ]; then
git clone -b releases/2.3 git://github.com/smuething/dune-multidomaingrid.git
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf dune-multidomaingrid
return
fi
cd $TOPDIR
}
installNLOPT()
{
cd $EXTDIR
rm -rf nlopt* standalone
if [ ! -e nlopt-2.4.2.tar.gz ]; then
wget http://ab-initio.mit.edu/nlopt/nlopt-2.4.2.tar.gz
fi
if test "$DOWNLOAD_ONLY" == "y"; then
return
fi
if test "$CLEANUP" == "y"; then
rm -rf nlopt
return
fi
mkdir nlopt
tar zxvf nlopt-2.4.2.tar.gz --strip-components=1 -C nlopt
cd nlopt
./configure
make
# show additional information
echo "In addition, it might be necessary to set manually"
echo "the nlopt path in the CMAKE_FLAGS section of the .opts-file:"
echo " -DNLOPT_ROOT=/path/to/nlopt \\"
cd $TOPDIR
}