Refactor offline

This refactoring includes some slicing and performance improvement attempts.

The slicing is still experimental and has some flaws:

• fancy indexing over thousands of elements might be horrible slow, eg f.events.tracks[idx_list, 0] where idx_list is a huge list of indices
• trivial problem of iterating: for track in f.evens.tracks does not stop after the last element
• meaningless indexing: f.events[5][23][420] is simply f.events[420] since the index/slice is not chained but overwritten when propagated. We need custom checks whether it makes sense
  • events are flat lists, so [x][y] does not make sense, it should throw an error
  • f.events[5:23][4] should effectively be the same as f.events[9] due to chaining of slice/index

km3io/offline.py

@@ -128,7 +128,6 @@ class OfflineReader:
 
 class Usr:
     """Helper class to access AAObject `usr`` stuff"""
-
     def __init__(self, mapper, branch, index=None):
         self._mapper = mapper
         self._name = mapper.name
@@ -159,8 +158,7 @@ class Usr:
         # to massively increase the performance. This needs triple check if
         # it's always the case.
         self._usr_names = [
-            n.decode("utf-8")
-            for n in self._branch['usr_names'].lazyarray(
+            n.decode("utf-8") for n in self._branch['usr_names'].lazyarray(
                 basketcache=BASKET_CACHE)[0]
         ]
         self._usr_idx_lookup = {
@@ -183,7 +181,9 @@ class Usr:
             n.decode("utf-8") for n in self.branch['usr_names'].lazyarray(
                 # TODO this will be fixed soon in uproot,
                 # see https://github.com/scikit-hep/uproot/issues/465
-                uproot.asgenobj(uproot.SimpleArray(uproot.STLVector(uproot.STLString())), self.branch['usr_names']._context, 6),
+                uproot.asgenobj(
+                    uproot.SimpleArray(uproot.STLVector(uproot.STLString())),
+                    self.branch['usr_names']._context, 6),
                 basketcache=BASKET_CACHE)[0]
         ]
         self.__getitem__ = self.__getitem_nested__
@@ -226,23 +226,31 @@ class Header:
     """The header"""
     def __init__(self, header):
         self._data = {}
-        self._missing_keys = set(header.keys()) - set(mc_header.keys())
 
-        for attribute, fields in mc_header.items():
-            values = header.get(attribute, '').split()
-            if not values:
+        for attribute, fields in header.items():
+            values = fields.split()
+            fields = mc_header.get(attribute, [])
+
+            n_values = len(values)
+            n_fields = len(fields)
+
+            if n_values == 1 and n_fields == 0:
+                self._data[attribute] = _to_num(values[0])
                 continue
+
+            n_max = max(n_values, n_fields)
+            values += [None] * (n_max - n_values)
+            fields += ["field_{}".format(i) for i in range(n_fields, n_max)]
+
             Constructor = namedtuple(attribute, fields)
-            if len(values) < len(fields):
-                values += [None] * (len(fields) - len(values))
+
+            if not values:
+                continue
+
             self._data[attribute] = Constructor(
                 **{f: _to_num(v)
                    for (f, v) in zip(fields, values)})
 
-        # quick fix while waiting for additional definitions in mc_header
-        for key in self._missing_keys:
-            self._data[key] = _to_num(header[key])
-
         for attribute, value in self._data.items():
             setattr(self, attribute, value)