Restructure package in order t load image on demand

.gitlab-ci.yml

@@ -12,22 +12,34 @@ cache:
   paths:
    - .cache/pip
    - venv/
+   - GiBUU.simg
   key: "$CI_COMMIT_REF_SLUG"
 
-
 stages:
     - test
     - coverage
     - docker
+    - reset_cache_image
     - release
 
+reset_test_image:
+    stage: reset_cache_image
+    cache:
+        paths:
+            - GiBUU.simg
+    script:
+        - rm GiBUU.simg
+    only:
+        - tags
+
 .virtualenv_template: &virtualenv_definition |
   python -V
   pip install virtualenv
   virtualenv venv
   source venv/bin/activate
   pip install -U pip setuptools
-  make install
+  echo n | make buildremote
+  make install-dev
 
 .junit_template: &junit_definition
     artifacts:

km3buu/__init__.py

-from os.path import isfile, abspath, join, dirname
+__author__ = "Johannes Schumann"
+__copyright__ = "Copyright 2020, Johannes Schumann and the KM3NeT collaboration."
+__credits__ = []
+__license__ = "MIT"
+__maintainer__ = "Johannes Schumann"
+__email__ = "jschumann@km3net.de"
+__status__ = "Development"
 
-MODULE_PATH = abspath(join(dirname(__file__), ".."))
-IMAGE_PATH = join(MODULE_PATH, "GiBUU.simg")
-
-if not isfile(IMAGE_PATH):
-    errmsg = "GiBUU image not found at %s;"
-    errmsg += "please run `make build` or `make buildremote`"
-    raise EnvironmentError(errmsg % IMAGE_PATH)
+IMAGE_NAME = "GiBUU.simg"
+DOCKER_URL = "docker://docker.km3net.de/simulation/km3buu:latest"

km3buu/config.py

+#!/usr/bin/env python
+# coding=utf-8
+# Filename: config.py
+# Author: Johannes Schumann <jschumann@km3net.de>
+"""
+
+
+"""
+import os
+import click
+from os.path import isfile, isdir, join, dirname, abspath
+from configparser import ConfigParser, Error, NoOptionError, NoSectionError
+from thepipe.logger import get_logger
+from . import IMAGE_NAME
+from .core import build_image
+
+__author__ = "Johannes Schumann"
+__copyright__ = "Copyright 2020, Johannes Schumann and the KM3NeT collaboration."
+__credits__ = []
+__license__ = "MIT"
+__maintainer__ = "Johannes Schumann"
+__email__ = "jschumann@km3net.de"
+__status__ = "Development"
+
+CONFIG_PATH = os.path.expanduser('~/.km3buu/config')
+
+log = get_logger(__name__)
+
+
+class Config(object):
+    def __init__(self, config_path=CONFIG_PATH):
+        self.config = ConfigParser()
+        self._config_path = config_path
+        os.makedirs(dirname(CONFIG_PATH), exist_ok=True)
+
+    def set(self, section, key, value):
+        if section not in self.config.sections():
+            self.config.add_section(section)
+        self.config.set(section, key, value)
+        with open(self._config_path, 'w') as f:
+            self.config.write(f)
+
+    def get(self, section, key, default=None):
+        try:
+            value = self.config.get(section, key)
+            try:
+                return float(value)
+            except ValueError:
+                return value
+        except (NoOptionError, NoSectionError):
+            return default
+
+    @property
+    def gibuu_image_path(self):
+        section = "GiBUU"
+        key = "image_path"
+        image_path = self.get(section, section)
+        if image_path is None or not isfile(image_path):
+            dev_path = abspath(join(dirname(__file__), os.pardir, IMAGE_NAME))
+            if isfile(dev_path):
+                image_path = dev_path
+            elif click.confirm("Is the GiBUU image already available?",
+                               default=False):
+                image_path = click.prompt("GiBUU image path?",
+                                          type=click.Path(exists=True,
+                                                          dir_okay=False))
+            elif click.confirm("Install image from remote?", default=True):
+                default_dir = join(os.environ["HOME"], ".km3buu")
+                image_dir = click.prompt(
+                    "GiBUU image path (default: ~/.km3buu) ?",
+                    type=click.Path(exists=True, file_okay=False),
+                    default=default_dir)
+                image_path = build_image(image_dir)
+            self.set(section, key, image_path)
+        return image_path
+
+    @gibuu_image_path.setter
+    def gibuu_image_path(self, value):
+        section = "GiBUU"
+        key = "image_path"
+        self.set(section, key, value)

km3buu/core.py

+# Filename: core.py
+"""
+Core functions for the package environment
+
+"""
+
+__author__ = "Johannes Schumann"
+__copyright__ = "Copyright 2020, Johannes Schumann and the KM3NeT collaboration."
+__credits__ = []
+__license__ = "MIT"
+__maintainer__ = "Johannes Schumann"
+__email__ = "jschumann@km3net.de"
+__status__ = "Development"
+
+import os
+from spython.main import Client
+from os.path import join, isdir, basename
+from thepipe.logger import get_logger
+
+from . import IMAGE_NAME, DOCKER_URL
+
+log = get_logger(basename(__file__))
+
+
+def build_image(output_dir):
+    if not isdir(output_dir):
+        raise OSError("Directory not found!")
+    image_path = join(output_dir, IMAGE_NAME)
+    return Client.build(DOCKER_URL, image=image_path, sudo=False, ext="simg")

km3buu/ctrl.py

@@ -14,15 +14,16 @@ __status__ = "Development"
 
 import os
 from spython.main import Client
-from os.path import join, abspath
-from tempfile import NamedTemporaryFile
-from . import MODULE_PATH, IMAGE_PATH
+from os.path import join, abspath, basename, isdir
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 from thepipe.logger import get_logger
 
-log = get_logger(__file__)
+from . import IMAGE_NAME
+from .config import Config
 
-INPUT_PATH = "/opt/buuinput2019/"
+log = get_logger(basename(__file__))
 
+INPUT_PATH = "/opt/buuinput2019/"
 GIBUU_SHELL = """
 #!/bin/bash
 
@@ -49,19 +50,26 @@ def run_jobcard(jobcard, outdir):
     outdir: str
         The path to the directory the output should be written to
     """
-    tmp_dir = join(MODULE_PATH, 'tmp')
-    os.makedirs(tmp_dir, exist_ok=True)
+    tmp_dir = TemporaryDirectory()
     outdir = abspath(outdir)
     log.info("Create temporary file for jobcard")
-    tmpfile_jobcard = NamedTemporaryFile(suffix='.job', dir=tmp_dir)
-    with open(tmpfile_jobcard.name, 'w') as f:
+    jobcard_fpath = join(tmp_dir.name, "tmp.job")
+    with open(jobcard_fpath, 'w') as f:
         f.write(str(jobcard))
     log.info("Create temporary file for associated runscript")
-    tmpfile_shell = NamedTemporaryFile(suffix='.sh', dir=tmp_dir)
-    with open(tmpfile_shell.name, 'w') as f:
-        ctnt = GIBUU_SHELL.format(outdir, tmpfile_jobcard.name)
+    script_fpath = join(tmp_dir.name, "run.sh")
+    with open(script_fpath, 'w') as f:
+        ctnt = GIBUU_SHELL.format(outdir, jobcard_fpath)
         f.write(ctnt)
-    output = Client.execute(IMAGE_PATH, ['/bin/sh', tmpfile_shell.name],
-                            bind=[outdir, tmp_dir])
-    log.info("GiBUU exited with return code {0}".format(output["return_code"]))
+    os.system("ls %s" % (tmp_dir.name))
+    output = Client.execute(Config().gibuu_image_path,
+                            ['/bin/sh', script_fpath],
+                            bind=[outdir, tmp_dir.name],
+                            return_result=True)
+    msg = output['message']
+    if isinstance(msg, str):
+        log.info("GiBUU output:\n %s" % msg)
+    else:
+        log.info("GiBUU output:\n %s" % msg[0])
+        log.error("GiBUU stacktrace:\n%s" % msg[1])
     return output["return_code"]

km3buu/jobcard.py

@@ -15,7 +15,7 @@ __status__ = "Development"
 INPUT_PATH = "/opt/buuinput2019/"
 
 PROCESS_LOOKUP = {"cc": 2, "nc": 3, "anticc": -2, "antinc": -3}
-FLAVOUR_LOOKUP = {"electron": 1, "muon": 2, "tau": 3}
+FLAVOR_LOOKUP = {"electron": 1, "muon": 2, "tau": 3}
 XSECTIONMODE_LOOKUP = {
     "integratedSigma": 0,
     "dSigmadCosThetadElepton": 1,
@@ -48,6 +48,7 @@ class Jobcard(object):
     def __init__(self, input_path=INPUT_PATH):
         self.input_path = "'%s'" % input_path
         self._groups = {}
+        self.set_property("input", "path_to_input", self.input_path)
 
     def set_property(self, group, name, value):
         """ Set a property to the jobcard

requirements.txt

 setuptools_scm
 spython
+thepipe
+click

setup.py

@@ -7,11 +7,7 @@ KM3BUU setup script.
 """
 import os
 import tempfile
-from os.path import dirname
 from setuptools import setup, find_packages
-from setuptools.command.install import install
-from setuptools.command.develop import develop
-from setuptools.command.egg_info import egg_info
 
 PACKAGE_NAME = 'km3buu'
 URL = 'https://git.km3net.de/simulation/km3buu'
@@ -22,35 +18,6 @@ __email__ = 'jschumann@km3net.de'
 with open('requirements.txt') as fobj:
     REQUIREMENTS = [l.strip() for l in fobj.readlines()]
 
-
-def _build_image():
-    ofile = tempfile.TemporaryFile()
-    retval = os.system("cd %s; make buildremote > %s" %
-                       (dirname(__file__), ofile.name))
-    if retval != 0:
-        with open(ofile.name, 'r') as f:
-            msg = f.read()
-            raise EnvironmentError(msg)
-
-
-class CustomInstallCmd(install):
-    def run(self):
-        install.run(self)
-        _build_image()
-
-
-class CustomDevelopCmd(develop):
-    def run(self):
-        develop.run(self)
-        _build_image()
-
-
-class CustomEggInfoCmd(egg_info):
-    def run(self):
-        egg_info.run(self)
-        _build_image()
-
-
 setup(
     name=PACKAGE_NAME,
     url=URL,
@@ -60,11 +27,6 @@ setup(
     packages=find_packages(),
     include_package_data=True,
     platforms='any',
-    cmdclass={
-        'install': CustomInstallCmd,
-        'develop': CustomDevelopCmd,
-        'egg_info': CustomEggInfoCmd
-    },
     setup_requires=['setuptools_scm'],
     use_scm_version={
         'write_to': '{}/version.txt'.format(PACKAGE_NAME),