installexternal.py 14.4 KB
Newer Older
1
2
3
4
5
6
#!/usr/bin/env python3

"""
install external stuff for dumux
"""
import os
7
8
import shutil
import re
9
import urllib.request
10
11
12
13
import tarfile
import sys
import subprocess
import argparse
14
import textwrap
15

16

17
# pylint: disable=C0103,W0212,W0622,C0116
18
class ChoicesAction(argparse._StoreAction):
19
20
    """Action to show choices in argparse"""

21
    def __init__(self, **kwargs):
22
        super().__init__(**kwargs)
23
24
25
        if self.choices is None:
            self.choices = []
        self._choices_actions = []
26
27

    def add_choice(self, choice, help=""):
28
29
30
        self.choices.append(choice)
        choice_action = argparse.Action(option_strings=[], dest=choice, help=help)
        self._choices_actions.append(choice_action)
31

32
33
34
    def _get_subactions(self):
        return self._choices_actions

35

36
37
38
39
40
# pylint: enable=C0103,W0212,W0622,C0116


def showMessage(message):
    """Pretty print for info MESSAGES"""
41
42
43
44
45
46
47
    print("*" * 120)
    print(message)
    print("")
    print("*" * 120)


if len(sys.argv) == 1:
48
    showMessage(
49
50
        "No options given. For more information "
        "run the following command: \n ./installexternal.py --help"
51
    )
52
53
    sys.exit()

54
55
56
57
58
59
60
parser = argparse.ArgumentParser(
    prog="installexternal",
    usage="./installexternal.py [OPTIONS] PACKAGES",
    description="This script downloads extenstions for dumux and dune \
                                     and installs some External Libraries and Modules.",
)
parser.register("action", "store_choice", ChoicesAction)
61
# Positional arguments
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
group = parser.add_argument_group(title="your choice of packages")
packages = group.add_argument("packages", nargs="+", metavar="PACKAGES", action="store_choice")
packages.add_choice("dumux-extensions", help="Download dumux-course and dumux-lecture.")
packages.add_choice(
    "dune-extensions",
    help="Download dune-uggrid, dune-alugrid, dune-foamgrid, \
                    dune-subgrid, dune-spgrid, dune-mmesh and dune-functions.",
)
packages.add_choice("optimization", help="Download and install glpk and nlopt.")
packages.add_choice("others", help="Download and install opm , metis and gstat.")
packages.add_choice("lecture", help="Download dumux-lecture.")
packages.add_choice("course", help="Download dumux-course.")
packages.add_choice("ug", help="Download dune-uggrid.")
packages.add_choice("alugrid", help="Download dune-alugrid.")
packages.add_choice("foamgrid", help="Download dune-foamgrid.")
packages.add_choice("subgrid", help="Download dune-subgrid.")
packages.add_choice("spgrid", help="Download dune-spgrid.")
packages.add_choice("mmesh", help="Download dune-mmesh.")
packages.add_choice("functions", help="Download dune-functions.")
packages.add_choice("glpk", help="Download and install glpk.")
packages.add_choice("nlopt", help="Download and install nlopt.")
packages.add_choice("opm", help="Download opm modules required for cornerpoint grids.")
packages.add_choice("metis", help="Download and install the METIS graph partitioner.")
packages.add_choice("gstat", help="Download and install gstat.")
86
87
88
89


# Optional arguments
options = parser.add_mutually_exclusive_group(required=False)
90
91
92
93
94
95
96
options.add_argument(
    "--clean", action="store_true", default=False, help="Delete all files for the given packages."
)
options.add_argument(
    "--download", action="store_true", default=False, help="Only download the packages."
)

97
98
99
100
parser.add_argument("--duneBranch", default="releases/2.7", help="Dune branch to be checked out.")
parser.add_argument("--dumuxBranch", default="releases/3.4", help="Dumux branch to be checked out.")
parser.add_argument("--opmBranch", default="release/2020.10", help="Opm branch to be checked out.")
parser.add_argument("--mmeshBranch", default="release/1.2", help="Mmesh branch to be checked out.")
101
102
103

args = vars(parser.parse_args())

104

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def runCommand(command, currentDir="."):
    """Helper function to run commands with error checking and reporting"""
    with open(currentDir + "/installexternal.log", "a") as log:
        with subprocess.Popen(command, stdout=log, stderr=log, universal_newlines=True) as popen:
            returnCode = popen.wait()
            if returnCode:
                message = textwrap.dedent(
                    f"""
                    (Error) The command {command} returned with non-zero exit code
                    If you can't fix the problem yourself consider reporting your issue
                    on the mailing list (dumux@listserv.uni-stuttgart.de) and
                    attach the file 'installexternal.log'
                """
                )
                showMessage(message)
                sys.exit(1)


def gitClone(url, branch=None):
    """Clone a repository from a given URL"""
125
126
127
    clone = ["git", "clone"]
    if branch:
        clone += ["-b", branch]
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    runCommand(command=[*clone, url])


def branchName(package, parameters):
    """Get the correct branch name"""
    # Set the branch
    if "dumux" in package:
        return parameters["dumux_branch"]
    if "mmesh" in package:
        return parameters["mmesh_branch"]
    if "dune" in package:
        return parameters["dune_branch"]
    if "opm" in package:
        return parameters["opm_branch"]
    return ""


def cleanPackage(package, finalMessage):
    """Clean up after a package"""
    if os.path.isfile(package + ".tar.gz"):
        os.remove(package + ".tar.gz")
    if os.path.exists(package):
        shutil.rmtree(package)
        finalMessage.append("{} has been removed.".format(package))
    else:
        # Save message to be shown at the end
        finalMessage.append("The folder {} does not exist.".format(package))


def filterPackageList(packageListOld):
    """Filter the package list and add possible dependencies"""
    packageList = []
    for pkg in packageListOld:
        if pkg in PACKAGE_NAMES:
            packageList.extend(PACKAGE_NAMES[pkg])
163
        else:
164
165
            packageList.extend([key for key in EXTERNAL_URLS if pkg in key])
    return packageList
166
167


168
169
def checkLocation():
    """check call location of this script"""
170
    if not os.path.isdir("dune-common"):
171
        showMessage(
172
173
            "You have to call " + sys.argv[0] + " for " + sys.argv[1] + " from\n"
            "the same directory in which dune-common is located.\n"
174
175
            "You cannot install it in this folder."
        )
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
        raise Exception("Script called in wrong location. Aborting.")


def installFromTarball(package, parameters, externalDir, finalMessage):
    """Install a package that uses a tarball as source code archive"""
    # Download the tarfile
    with urllib.request.urlopen(EXTERNAL_URLS[package]) as fileData:
        dataToWrite = fileData.read()
        with open(externalDir + "/" + package + ".tar.gz", "wb") as file:
            file.write(dataToWrite)

    # Save message to be shown at the end
    finalMessage.append("{} has been successfully downloaded.".format(package))

    # Start Installation if the flag download is set to false.
    if not parameters["download"]:
        # Extract
        with tarfile.open(package + ".tar.gz") as tarArchive:
            tarArchive.extractall()
            shutil.move(os.path.commonprefix(tarArchive.getnames()), package)  # rename

        # Start the configuration
        os.chdir(externalDir + "/" + package)
        if package == "gstat":
            with open("configure", "r+") as file:
                content = file.read()
                file.seek(0)
                file.truncate()
                file.write(content.replace("doc/tex/makefile", ""))

        # Run Configuration command
        configCmd = "./configure" if package != "metis" else ["make", "config"]
        runCommand(configCmd, currentDir=externalDir)
        try:
            runCommand("make", currentDir=externalDir)
        except subprocess.CalledProcessError as exc:
            raise Exception("{} installation has failed.".format(package)) from exc
        # Save message to be shown at the end
        if os.path.exists(externalDir + "/" + package):
            finalMessage.append("{} has been successfully installed.".format(package))


def installExternal(parameters):
    """Main driver: install external packages"""

    topDir = os.getcwd()
    externalDir = topDir + "/external"
    parameters["packages"] = filterPackageList(parameters["packages"])

    # print the list of packages to be downloaded/installed/removed
    print(
        "The following package(s) will be {0}:\n".format(
            "removed" if parameters["clean"] else "downloaded"
        ),
        ", ".join(parameters["packages"]),
        "\n",
    )

    checkLocation()
235
236

    # clear the log file
237
238
239
    logDir = externalDir if os.path.exists(externalDir) else topDir
    with open(logDir + "/installexternal.log", "w") as _:
        pass
240

241
242
243
    finalMessage = []
    for package in parameters["packages"]:
        os.chdir(topDir)
244
        # Package name for final message
245
246
247
248
249
250
251
252
        finalMessage.append("[---" + package + "---]")

        # Set the directory: create externalDir for external packages
        if not any(re.compile(p).match(package) for p in ["dumux", "dune", "opm"]):
            os.makedirs(externalDir, exist_ok=True)
            os.chdir(externalDir)

        branch = branchName(package, parameters)
253
254

        # Run the requested command
255
256
257
        if parameters["clean"]:
            cleanPackage(package, finalMessage)
            continue
258

259
260
261
262
263
264
        # Check if tarball
        tarball = EXTERNAL_URLS[package].endswith("tar.gz")

        if not os.path.exists(package):
            if tarball:
                installFromTarball(package, parameters, externalDir, finalMessage)
265
            else:
266
267
                # Clone from repo
                gitClone(EXTERNAL_URLS[package], branch)
268
                # Save message to be shown at the end
269
                finalMessage.append("{} has been sucessfully cloned.".format(package))
270
        else:
271
272
            if tarball:
                finalMessage.append("{} has been already installed.".format(package))
273
            else:
274
275
276
                # Checkout to the requested branch
                os.chdir(topDir + "/" + package)
                with subprocess.Popen(["git", "checkout", branch]) as _:
277
                    # Save message to be shown at the end
278
                    finalMessage.append(
279
280
                        "-- Skip cloning {}, because the folder already exists.".format(package)
                    )
281
                    finalMessage.append("-- Checking out {} ".format(package) + branch)
282
283
284
                    continue

        # Save post installation message if there is any.
285
286
        if package in MESSAGES.keys():
            finalMessage.extend(MESSAGES[package])
287

288
289
        # Change to topDir
        os.chdir(topDir)
290
291

    # Save post installation message about dunecontrol if need be.
292
293
294
295
    if not parameters["clean"] and any(
        x in pkg for pkg in parameters["packages"] for x in ["dumux", "dune", "opm"]
    ):
        finalMessage.append(
296
297
298
            "\n\nPlease run the following command "
            "(can be copied to command line):\n\n  "
            "./dune-common/bin/dunecontrol --opts=./dumux/cmake.opts all"
299
        )
300
301

    # If cleanup and only logfile in the external directory, remove the directory
302
303
304
305
    if os.path.isdir(externalDir):
        _, _, files = next(os.walk(externalDir))
        if parameters["clean"] and len(files) == 1 and "installexternal.log" in files:
            shutil.rmtree(externalDir)
306

307
    return "\n".join(finalMessage)
308

309
310
311

#################################################################
#################################################################
312
# (1/2) Define the necessary packages and their URLS
313
314
#################################################################
#################################################################
315
316
317
318
319
320
321
322
DUNE_GIT_BASEURL = "https://gitlab.dune-project.org/"
DUMUX_GIT_BASEURL = "https://git.iws.uni-stuttgart.de/dumux-repositories/"
EXTERNAL_URLS = {
    "dumux-lecture": DUMUX_GIT_BASEURL + "dumux-lecture.git",
    "dumux-course": DUMUX_GIT_BASEURL + "dumux-course.git",
    "dune-uggrid": DUNE_GIT_BASEURL + "/staging/dune-uggrid.git",
    "dune-alugrid": DUNE_GIT_BASEURL + "extensions/dune-alugrid.git",
    "dune-foamgrid": DUNE_GIT_BASEURL + "extensions/dune-foamgrid.git",
323
    "dune-subgrid": "https://git.imp.fu-berlin.de/agnumpde/dune-subgrid.git",
324
325
326
327
    "dune-spgrid": DUNE_GIT_BASEURL + "extensions/dune-spgrid.git",
    "dune-mmesh": DUNE_GIT_BASEURL + "samuel.burbulla/dune-mmesh.git",
    "dune-functions": DUNE_GIT_BASEURL + "staging/dune-functions.git",
    "dune-typetree": DUNE_GIT_BASEURL + "staging/dune-typetree.git",
328
329
330
331
332
333
334
335
    "glpk": "http://ftp.gnu.org/gnu/glpk/glpk-4.60.tar.gz",
    "nlopt": "http://ab-initio.mit.edu/nlopt/nlopt-2.4.2.tar.gz",
    "opm-common": "https://github.com/OPM/opm-common",
    "opm-grid": "https://github.com/OPM/opm-grid",
    "metis": "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz",
    "gstat": "http://gstat.org/gstat.tar.gz",
}

336
PACKAGE_NAMES = {
337
    "dumux-extensions": ["dumux-lecture", "dumux-course"],
338
339
340
341
342
343
344
345
346
347
    "dune-extensions": [
        "dune-uggrid",
        "dune-alugrid",
        "dune-foamgrid",
        "dune-subgrid",
        "dune-spgrid",
        "dune-mmesh",
        "dune-functions",
        "dune-typetree",
    ],
348
349
    "functions": ["dune-functions", "dune-typetree"],
    "optimization": ["glpk", "nlopt"],
350
    "others": ["opm-common", "opm-grid", "metis", "gstat"],
351
352
}

353
MESSAGES = {
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
    "glpk": [
        "In addition, it might be necessary to set manually",
        "the glpk path in the CMAKE_FLAGS section of the .opts-file:",
        "  -DGLPK_ROOT=/path/to/glpk \\",
    ],
    "dune-mmesh": [
        "Maybe you also have to install CGAL",
        "(see cgal.org/download.html)",
        "Maybe you also need to change your core dune modules' branches to their newest versions.",
    ],
    "opm-common": [
        "In addition, it might be necessary to set manually some",
        "CMake variables in the CMAKE_FLAGS section of the .opts-file:",
        "  -DUSE_MPI=ON",
        "Currently, compiling opm with clang is not possible.",
        "",
        "Maybe you also have to install the following packages (see the",
        " opm prerequisites at opm-project.org):",
        "  BLAS, LAPACK, Boost, SuiteSparse, Zoltan",
    ],
374
375
376
377
378
}


#################################################################
#################################################################
379
# (2/2) Download/config/clean the requested packages
380
381
382
#################################################################
#################################################################
# Start download/configuration/cleaning tasks
383
showMessage(installExternal(args))