Merge branch 'feature/python-createdockerimage-script' into 'master'

Feature/python createdockerimage script

See merge request !2231

bin/util/createdockerimage.py

+#!/usr/bin/env python3
+import os
+import sys
+import string
+import shutil
+import argparse
+import subprocess
+from getmoduleinfo import getModuleFile
+from getmoduleinfo import extractModuleInfos
+
+# require python 3
+if sys.version_info[0] < 3:
+    sys.exit("\nERROR: Python3 required")
+
+# input argument parser
+parser = argparse.ArgumentParser(
+    description="Create a docker image for a given module and install script."
+)
+
+parser.add_argument('-m', '--modulepath',
+                    required=True,
+                    help='the path to the your module')
+parser.add_argument('-i', '--installScript',
+                    required=True,
+                    help="Specify the installation script")
+parser.add_argument('-t', '--templateFolder',
+                    required=False,
+                    help="Specify the folder with the template files")
+
+args = vars(parser.parse_args())
+
+# get information on the module
+modulePath = os.path.abspath(args['modulepath'])
+modInfo = extractModuleInfos(getModuleFile(modulePath),
+                             ['Module', 'Maintainer'])
+moduleName = modInfo['Module']
+moduleMaintainer = modInfo['Maintainer']
+dockerTag = moduleName.lower()  # docker only supports lower case
+
+# get folder with the template files
+templateFolder = args['templateFolder']
+if not templateFolder:
+    templateFolder = os.path.join(modulePath, '../dumux/docker')
+if not os.path.exists(templateFolder):
+    sys.exit("Template folder {} could not be found".format(templateFolder))
+
+print("*"*54)
+print("\n-- Creating a Docker image for module " + moduleName + " --\n")
+print("*"*54)
+
+if os.path.exists("docker"):
+    print("\nA docker folder already exists. "
+          "Continue anyway? - will be overwritten - [y/N]\n")
+    delete = input()
+    if delete == "y" or delete == "Y":
+        shutil.rmtree("docker")
+        print("--> Deleted old docker folder.")
+    else:
+        sys.exit("Abort.")
+
+os.mkdir("docker")
+print("--> Created the folder 'docker'.")
+
+# copy install script into docker folder and make it executable
+installScriptPath = args['installScript']
+installScriptName = os.path.split(installScriptPath)[1]
+installScript = os.path.join(os.path.join(os.getcwd(), 'docker'),
+                             installScriptName)
+shutil.copy(installScriptPath, installScript)
+os.system("chmod +x {}".format(installScript))
+print("--> Using install script: {} to install dependencies for module {}."
+      .format(installScript, moduleName))
+
+
+# substitute content from template and write to target
+def substituteAndWrite(template, target, mapping):
+    if not os.path.exists(template):
+        sys.exit("Template file '" + template + "' could not be found")
+    with open(target, 'w') as targetFile:
+        raw = string.Template(open(template).read())
+        targetFile.write(raw.substitute(**mapping))
+
+
+# write setpermissions helper script
+template = os.path.join(templateFolder, 'setpermissions.sh.template')
+target = os.path.join(os.getcwd(), 'docker/setpermissions.sh')
+substituteAndWrite(template, target, {})
+print("--> Created permission helper script for easier container setup.")
+
+# write welcome message file
+template = os.path.join(templateFolder, 'WELCOME.template')
+target = os.path.join(os.getcwd(), 'docker/WELCOME')
+substituteAndWrite(template, target,
+                   {'modName': moduleName, 'modFolder': moduleName})
+print("--> Created welcome message displayed on Docker container startup.")
+
+# write readme file
+template = os.path.join(templateFolder, 'README.md.template')
+target = os.path.join(os.getcwd(), 'docker/README.md')
+substituteAndWrite(template, target,
+                   {'modName': moduleName, 'dockerTag': dockerTag})
+print("--> Created README.md on how to use the docker image.")
+
+# write helper file for container spin-up (make it executable after creation)
+template = os.path.join(templateFolder, 'docker.sh.template')
+target = os.path.join(os.getcwd(), 'docker/docker_{}.sh'.format(dockerTag))
+substituteAndWrite(template, target, {'dockerTag': dockerTag})
+os.system("chmod +x " + target)
+print("--> Created helper script to spin up the docker container.")
+
+# write the docker file
+template = os.path.join(templateFolder, 'Dockerfile.template')
+target = os.path.join(os.getcwd(), 'docker/Dockerfile')
+substituteAndWrite(template, target,
+                   {
+                       'modName': moduleName,
+                       'modMaintainer': moduleMaintainer,
+                       'dockerTag': dockerTag,
+                       'instScript': installScriptName
+                   })
+print("--> Created Dockerfile. You can adapt it to your needs.")
+print()
+print("Do you want to directly build the Docker image? [y/N]")
+
+build = input()
+if build == "y" or build == "Y":
+    print("Building Docker image... this may take several minutes.")
+    try:
+        os.chdir('docker')
+        subprocess.run(['docker', 'build',
+                        '-f', 'Dockerfile',
+                        '-t', dockerTag, '.'], check=True)
+        os.chdir('../')
+    except Exception:
+        os.chdir('../')
+        sys.exit("ERROR: docker image build failed")
+
+    print()
+    print("Successfully built image: {}. "
+          "Have a look at docker/README.md.".format(dockerTag))
+    print("Check the container by running "
+          "'docker run -it {} /bin/bash' in the same".format(dockerTag))
+    print("directory as the Dockerfile, and try using the convenience script "
+          "docker_{}.sh".format(dockerTag))
+    print("See docker/README.md for more information.")
+else:
+    print("You can build your Docker image later by running "
+          "'docker build -f Dockerfile -t {}'".format(dockerTag))
+    print("from within the folder 'docker' that was created by this script, "
+          "and in which you should find the 'Dockerfile'.")

docker/Dockerfile.template

+# ${modName} docker container
+# see https://github.com/phusion/baseimage-docker for information on the base image
+# It is Ubuntu LTS customized for better Docker compatibility
+FROM phusion/baseimage:18.04-1.0.0
+MAINTAINER ${modMaintainer}
+
+# run Ubuntu update as advised on https://github.com/phusion/baseimage-docker
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get upgrade -y -o Dpkg::Options::="--force-confold" \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# install the basic dependencies
+RUN apt-get update \
+    && apt-get install --no-install-recommends --yes \
+    ca-certificates \
+    vim \
+    python3-dev \
+    python3-pip \
+    git \
+    pkg-config \
+    build-essential \
+    gfortran \
+    mpi-default-bin \
+    mpi-default-dev \
+    libsuitesparse-dev \
+    libsuperlu-dev \
+    libeigen3-dev \
+    doxygen \
+    wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# get cmake 3.18 and export it to path
+RUN mkdir -p cmake-3.18
+RUN wget -qO- "https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C cmake-3.18
+ENV PATH="/cmake-3.18/bin:$${PATH}"
+
+# add the permission helper script to the my_init service
+COPY setpermissions.sh /etc/my_init.d/setpermissions.sh
+
+# create a dumux user
+# add the welcome message (copied further down) output to bashrc
+# make the set permission helper script executable
+# add user to video group which enables graphics if desired
+RUN useradd -m --home-dir /dumux dumux \
+    && echo "cat /dumux/WELCOME" >> /dumux/.bashrc \
+    && chmod +x /etc/my_init.d/setpermissions.sh \
+    && usermod -a -G video dumux
+
+# switch to the dumux user and set the working directory
+USER dumux
+WORKDIR /dumux
+
+# create a shared volume communicating with the host
+RUN mkdir /dumux/shared
+VOLUME /dumux/shared
+
+# This is the message printed on entry
+COPY WELCOME /dumux/WELCOME
+
+# set git user in case installation requires to apply patches
+RUN git config --global user.name "${modName}"
+RUN git config --global user.email "${modMaintainer}"
+
+# Install the dumux module and its dependencies
+# This expects the install script to do everything from clone to configure
+COPY ${instScript} /dumux/${instScript}
+RUN ./${instScript} && rm -f /dumux/${instScript}
+
+# unset git user
+RUN git config --global --unset user.name
+RUN git config --global --unset user.email
+
+# switch back to root
+WORKDIR /dumux
+USER root
+
+# set entry point like advised https://github.com/phusion/baseimage-docker
+# this sets the permissions right, see above
+ENTRYPOINT ["/sbin/my_init","--quiet","--","/sbin/setuser","dumux","/bin/bash","-l","-c"]
+
+# start interactive shell
+CMD ["/bin/bash","-i"]

docker/README.md.template

+# Readme for the dumux module ${modName}
+
+You created the Docker image ${dockerTag}. Next steps:
+
+* Try your container by running docker_${dockerTag}.sh open
+  See below for instructions how to share files with the host system.
+* Push the docker image to DockerHub or the GitLab Docker registry of your dumux module.
+  Look at the Registry tab of your dumux module for help.
+
+* Replace the image name in docker_${dockerTag}.sh with the actual image name
+  e.g. git.iws.uni-stuttgart.de:4567/dumux-pub/koch2017a.
+
+* [Optional] Add the Dockerfile to the git repository.
+
+* Add the docker_${dockerTag}.sh script to the git repository (this is for the user)
+  and add the following lines to your README.md:
+
+Using dumux module ${modName} with docker
+=============================================
+
+In order to run simulations of this module look
+at the convenience script docker_${dockerTag}.sh.
+First download the script from the git repository.
+
+The simplest way is to spin up a container
+is creating a new folder "dumux"
+$$ mkdir dumux
+change to the new folder
+$$ cd dumux
+and open the module image by running
+$$ docker_${dockerTag}.sh open
+
+The container will spin up. It will mount the "dumux"
+directory into the container at /dumux/shared. Put files
+in this folder to share them with the host machine. This
+could be e.g. VTK files produced by the simulation that
+you want to visualize on the host machine.

docker/WELCOME.template

+
+*****************************************************************
+Welcome to the Docker container of the dumux module "${modName}"!
+
+You can compile and execute the examples from within the build tree of the
+module at /dumux/PATH_TO/${modFolder}/build-cmake.
+
+Note that the container does not have graphics support, but copying the output
+(e.g. VTK files) to the folder /dumux/shared will make them available outside
+this container on the host for display.

docker/docker.sh.template

+#!/usr/bin/env bash
+
+# TODO set the image name here to a global address
+#      after adding your container to your modules container registry
+# e.g. git.iws.uni-stuttgart.de:4567/dumux-pub/koch2017a
+IMAGE_NAME=${dockerTag}
+
+# the host directory ...
+SHARED_DIR_HOST="$$(pwd)"
+# ... that is mounted into this container directory:
+SHARED_DIR_CONTAINER="/dumux/shared"
+
+help ()
+{
+    echo ""
+    echo "Usage: docker_${dockerTag}.sh <command> [options]"
+    echo ""
+    echo "  docker_${dockerTag}.sh open [image]      - run a container from the image."
+    echo "  docker_${dockerTag}.sh help              - display this message."
+    echo ""
+    echo "Optionally supply a Docker image name to the open command."
+    echo ""
+}
+
+# start a container. Only argument is the Docker image.
+open()
+{
+    IMAGE="$$1"
+    docker run -it \
+                -e HOST_UID=$$(id -u $$USER) \
+                -e HOST_GID=$$(id -g $$USER) \
+                -v $$SHARED_DIR_HOST:$$SHARED_DIR_CONTAINER \
+                --name dumux_${dockerTag} \
+                $$IMAGE /bin/bash
+}
+
+# Check if user specified valid command otherwise print help message
+if [ "$$1" == "open" ]; then
+    IMAGE="$$2" : $${IMAGE:="$$IMAGE_NAME"}
+    open $$IMAGE
+else
+    help
+    exit 1
+fi

docker/setpermissions.sh.template

+#!/usr/bin/env bash
+
+echo "Setting permissions for shared folder"
+
+# if HOST_UID or HOST_GID are passed to the container
+# as environment variables, e.g. by calling
+# docker run -e HOST_UID=$$(id -u $$USER) -e HOST_GID=$$(id -g $$USER),
+# then we set the permissions of the files in the shared folder
+if [ "$$HOST_UID" ]; then
+    echo "Changing user id to the provided one"
+    usermod -u $$HOST_UID dumux
+fi
+if [ "$$HOST_GID" ]; then
+    echo "Changing group id to the provided one"
+    groupmod -g $$HOST_GID dumux
+fi
+
+# Change permissions only if both user and group id were passed.
+# Otherwise, this would change ownership to the default id of dumux,
+# which could lead to permission issues with the host user.
+if [ "$${HOST_UID}" -a "$${HOST_GID}" ]; then
+    # find all data in /dumux/shared/ and transfer ownership.
+    # sed "1d" removes the /dumux/shared folder itself (first line) that
+    # is within the results of the find command. If no files are present,
+    # chown returns an error because arguments are missing. Therefore, errors
+    # are redirected into /dev/null. Still, the script might return with an error
+    # in this case, and we guarantee successful execution with the || true trick at the end
+    find /dumux/shared/ | sed "1d" | xargs chown -R dumux:dumux 2> /dev/null || true
+else
+    echo "Skipping ownership transfer as host user and/or group id were not provided"
+fi