WIP: Slicing and refactoring offline

Closes #37 (closed)

km3io/offline.py

 from collections import namedtuple
 import uproot
 import numpy as np
-import awkward as ak
 import warnings
 from .definitions import mc_header
 
 MAIN_TREE_NAME = "E"
 # 110 MB based on the size of the largest basket found so far in km3net
 BASKET_CACHE_SIZE = 110 * 1024**2
+BASKET_CACHE = uproot.cache.ThreadSafeArrayCache(BASKET_CACHE_SIZE)
+EXCLUDE_KEYS = set(["AAObject", "t", "fBits", "fUniqueID"])
 
-
-BranchMapper = namedtuple("BranchMapper", ['name', 'key', 'extra', 'exclude', 'update', 'attrparser'])
+BranchMapper = namedtuple(
+    "BranchMapper",
+    ['name', 'key', 'extra', 'exclude', 'update', 'attrparser'])
 
 
 def _nested_mapper(key):
@@ -18,13 +20,57 @@ def _nested_mapper(key):
     return '_'.join(key.split('.')[1:])
 
 
-EXCLUDE_KEYS = set(["AAObject", "t", "fBits", "fUniqueID"])
-BRANCH_MAPS = [
-    BranchMapper("tracks", "trks", {}, ['trks.usr_data', 'trks.usr'], {}, _nested_mapper),
-    BranchMapper("mc_tracks", "mc_trks", {}, ['mc_trks.usr_data', 'mc_trks.usr'], {}, _nested_mapper),
-    BranchMapper("hits", "hits", {}, ['hits.usr'], {}, _nested_mapper),
-    BranchMapper("mc_hits", "mc_hits", {}, ['mc_hits.usr'], {}, _nested_mapper),
-    BranchMapper("events", "Evt", {'t_sec': 't.fSec', 't_ns': 't.fNanoSec'}, [], {'n_hits': 'hits', 'n_mc_hits': 'mc_hits', 'n_tracks': 'trks', 'n_mc_tracks': 'mc_trks'}, lambda a: a),
+EVENTS_MAP = BranchMapper(name="events",
+                          key="Evt",
+                          extra={
+                              't_sec': 't.fSec',
+                              't_ns': 't.fNanoSec'
+                          },
+                          exclude=[],
+                          update={
+                              'n_hits': 'hits',
+                              'n_mc_hits': 'mc_hits',
+                              'n_tracks': 'trks',
+                              'n_mc_tracks': 'mc_trks'
+                          },
+                          attrparser=lambda a: a)
+
+SUBBRANCH_MAPS = [
+    BranchMapper(
+        name="tracks",
+        key="trks",
+        extra={},
+        exclude=['trks.usr_data', 'trks.usr', 'trks.fUniqueID', 'trks.fBits'],
+        update={},
+        attrparser=_nested_mapper),
+    BranchMapper(name="mc_tracks",
+                 key="mc_trks",
+                 extra={},
+                 exclude=[
+                     'mc_trks.usr_data', 'mc_trks.usr', 'mc_trks.rec_stages',
+                     'mc_trks.fitinf', 'mc_trks.fUniqueID', 'mc_trks.fBits'
+                 ],
+                 update={},
+                 attrparser=_nested_mapper),
+    BranchMapper(name="hits",
+                 key="hits",
+                 extra={},
+                 exclude=[
+                     'hits.usr', 'hits.pmt_id', 'hits.origin', 'hits.a',
+                     'hits.pure_a', 'hits.fUniqueID', 'hits.fBits'
+                 ],
+                 update={},
+                 attrparser=_nested_mapper),
+    BranchMapper(name="mc_hits",
+                 key="mc_hits",
+                 extra={},
+                 exclude=[
+                     'mc_hits.usr', 'mc_hits.dom_id', 'mc_hits.channel_id',
+                     'mc_hits.tdc', 'mc_hits.tot', 'mc_hits.trig',
+                     'mc_hits.fUniqueID', 'mc_hits.fBits'
+                 ],
+                 update={},
+                 attrparser=_nested_mapper),
 ]
 
 
@@ -42,7 +88,7 @@ class cached_property:
 
 class OfflineReader:
     """reader for offline ROOT files"""
-    def __init__(self, file_path=None, fobj=None, data=None, index=slice(None)):
+    def __init__(self, file_path=None, fobj=None, data=None, index=None):
         """ OfflineReader class is an offline ROOT file wrapper
 
         Parameters
@@ -56,16 +102,18 @@ class OfflineReader:
         if file_path is not None:
             self._fobj = uproot.open(file_path)
             self._tree = self._fobj[MAIN_TREE_NAME]
-            self._data = self._tree.lazyarrays(
-                basketcache=uproot.cache.ThreadSafeArrayCache(
-                    BASKET_CACHE_SIZE))
+            self._data = self._tree.lazyarrays(basketcache=BASKET_CACHE)
         else:
             self._fobj = fobj
             self._tree = self._fobj[MAIN_TREE_NAME]
             self._data = data
 
-        for mapper in BRANCH_MAPS:
-            setattr(self, mapper.name, BranchElement(self._tree, mapper=mapper, index=self._index))
+    @cached_property
+    def events(self):
+        return Branch(self._tree,
+                      mapper=EVENTS_MAP,
+                      index=self._index,
+                      subbranchmaps=SUBBRANCH_MAPS)
 
     @classmethod
     def from_index(cls, source, index):
@@ -78,7 +126,9 @@ class OfflineReader:
         index: index or slice
             The index or slice to create the subtree.
         """
-        instance = cls(fobj=source._fobj, data=source._data[index], index=index)
+        instance = cls(fobj=source._fobj,
+                       data=source._data[index],
+                       index=index)
         return instance
 
     def __getitem__(self, index):
@@ -86,12 +136,10 @@ class OfflineReader:
 
     def __len__(self):
         tree = self._fobj[MAIN_TREE_NAME]
-        if self._index == slice(None):
+        if self._index is None:
             return len(tree)
         else:
-            return len(tree.lazyarrays(
-                basketcache=uproot.cache.ThreadSafeArrayCache(
-                    BASKET_CACHE_SIZE))[self.index])
+            return len(tree.lazyarrays(basketcache=BASKET_CACHE)[self.index])
 
     @cached_property
     def header(self):
@@ -104,10 +152,6 @@ class OfflineReader:
         else:
             warnings.warn("Your file header has an unsupported format")
 
-    @cached_property
-    def usr(self):
-        return Usr(self._tree)
-
     def get_best_reco(self):
         """returns the best reconstructed track fit data. The best fit is defined
         as the track fit with the maximum reconstruction stages. When "nan" is
@@ -401,10 +445,10 @@ class OfflineReader:
             are not found, None is returned as the stages index.
         """
         if mc is False:
-            stages_data = self.tracks.rec_stages
+            stages_data = self.events.tracks.rec_stages
 
         if mc is True:
-            stages_data = self.mc_tracks.rec_stages
+            stages_data = self.events.mc_tracks.rec_stages
 
         for trk_index, rec_stages in enumerate(stages_data):
             try:
@@ -448,14 +492,19 @@ class OfflineReader:
 
 
 class Usr:
-    """Helper class to access AAObject usr stuff"""
-    def __init__(self, tree):
+    """Helper class to access AAObject `usr`` stuff"""
+    def __init__(self, name, tree, index=None):
         # Here, we assume that every event has the same names in the same order
         # to massively increase the performance. This needs triple check if it's
         # always the case; the usr-format is simply a very bad design.
+        self._name = name
         try:
+            tree['usr']  # This will raise a KeyError in old aanet files
+            # which has a different strucuter and key (usr_data)
+            # We do not support those...
             self._usr_names = [
-                n.decode("utf-8") for n in tree['Evt']['usr_names'].array()[0]
+                n.decode("utf-8") for n in tree['usr_names'].lazyarray(
+                    basketcache=BASKET_CACHE)[0]
             ]
         except (KeyError, IndexError):  # e.g. old aanet files
             self._usr_names = []
@@ -464,9 +513,10 @@ class Usr:
                 name: index
                 for index, name in enumerate(self._usr_names)
             }
-            self._usr_data = tree['Evt']['usr'].lazyarray(
-                basketcache=uproot.cache.ThreadSafeArrayCache(
-                    BASKET_CACHE_SIZE))
+            data = tree['usr'].lazyarray(basketcache=BASKET_CACHE)
+            if index is not None:
+                data = data[index]
+            self._usr_data = data
             for name in self._usr_names:
                 setattr(self, name, self[name])
 
@@ -482,11 +532,14 @@ class Usr:
             entries.append("{}: {}".format(name, self[name]))
         return '\n'.join(entries)
 
+    def __repr__(self):
+        return "<{}[{}]>".format(self.__class__.__name__, self._name)
+
 
 def _to_num(value):
-    """Convert value to a numerical value if possible"""
+    """Convert a value to a numerical one if possible"""
     if value is None:
-        return None
+        return
     try:
         return int(value)
     except ValueError:
@@ -499,7 +552,7 @@ def _to_num(value):
 
 
 class Header:
-    """The online header"""
+    """The header"""
     def __init__(self, header):
         self._data = {}
         for attribute, fields in mc_header.items():
@@ -509,7 +562,9 @@ class Header:
             Constructor = namedtuple(attribute, fields)
             if len(values) < len(fields):
                 values += [None] * (len(fields) - len(values))
-            self._data[attribute] = Constructor(**{f: _to_num(v) for (f, v) in zip(fields, values)})
+            self._data[attribute] = Constructor(
+                **{f: _to_num(v)
+                   for (f, v) in zip(fields, values)})
 
         for attribute, value in self._data.items():
             setattr(self, attribute, value)
@@ -521,56 +576,104 @@ class Header:
         return "\n".join(lines)
 
 
-class BranchElement:
-    """wrapper for offline tracks"""
-    def __init__(self, tree, mapper, index=slice(None)):
+class Branch:
+    """Branch accessor class"""
+    def __init__(self,
+                 tree,
+                 mapper,
+                 index=None,
+                 subbranchmaps=None,
+                 keymap=None):
         self._tree = tree
         self._mapper = mapper
         self._index = index
         self._keymap = None
-
         self._branch = tree[mapper.key]
+        self._subbranches = []
 
-        self._initialise_keys()
+        if keymap is None:
+            self._initialise_keys()  #
+        else:
+            self._keymap = keymap
+
+        if subbranchmaps is not None:
+            for mapper in subbranchmaps:
+                subbranch = Branch(self._tree,
+                                   mapper=mapper,
+                                   index=self._index)
+                self._subbranches.append(subbranch)
+        for subbranch in self._subbranches:
+            setattr(self, subbranch._mapper.name, subbranch)
 
     def _initialise_keys(self):
-        """Create the keymap and instance attributes"""
-        keys = set(k.decode('utf-8') for k in self._branch.keys()) - set(self._mapper.exclude) - EXCLUDE_KEYS
-        self._keymap = {**{self._mapper.attrparser(k): k for k in keys}, **self._mapper.extra}
+        """Create the keymap and instance attributes for branch keys"""
+        # TODO: this could be a cached property
+        keys = set(k.decode('utf-8') for k in self._branch.keys()) - set(
+            self._mapper.exclude) - EXCLUDE_KEYS
+        self._keymap = {
+            **{self._mapper.attrparser(k): k
+               for k in keys},
+            **self._mapper.extra
+        }
         self._keymap.update(self._mapper.update)
         for k in self._mapper.update.values():
             del self._keymap[k]
 
-        # self._EntryType = namedtuple(mapper.name[:-1], self.keys())
-
-        for key in self.keys():
-            # print("setting", self._mapper.name, key)
-            setattr(self, key, self[key])
+        for key in self._keymap.keys():
+            setattr(self, key, None)
 
     def keys(self):
         return self._keymap.keys()
 
+    @cached_property
+    def usr(self):
+        return Usr(self._mapper.name, self._branch, index=self._index)
+
+    def __getattribute__(self, attr):
+        if attr.startswith("_"):  # let all private and magic methods pass
+            return object.__getattribute__(self, attr)
+
+        if attr in self._keymap.keys():  # intercept branch key lookups
+            out = self._branch[self._keymap[attr]].lazyarray(
+                basketcache=BASKET_CACHE)
+            if self._index is not None:
+                out = out[self._index]
+            return out
+
+        return object.__getattribute__(self, attr)
+
     def __getitem__(self, item):
-        """Slicing magic a la numpy"""
-        if isinstance(item, slice):
-            return self.__class__(self._tree, self._mapper, index=item)
-        if isinstance(item, int):
-            return {
-                key: self._branch[self._keymap[key]].array()[self._index, item] for key in self.keys()
-                }
-        return self._branch[self._keymap[item]].lazyarray(
-                basketcache=uproot.cache.ThreadSafeArrayCache(
-                    BASKET_CACHE_SIZE))[self._index]
+        """Slicing magic"""
+        if isinstance(item, (int, slice)):
+            return self.__class__(self._tree,
+                                  self._mapper,
+                                  index=item,
+                                  keymap=self._keymap,
+                                  subbranchmaps=SUBBRANCH_MAPS)
+
+        if isinstance(item, tuple):
+            return self[item[0]][item[1]]
+
+        return self.__class__(self._tree,
+                              self._mapper,
+                              index=np.array(item),
+                              keymap=self._keymap,
+                              subbranchmaps=SUBBRANCH_MAPS)
 
     def __len__(self):
-        if self._index == slice(None):
+        if self._index is None:
             return len(self._branch)
+        elif isinstance(self._index, int):
+            return 1
         else:
-            return len(self._branch[self._keymap['id']].lazyarray()[self._index])
+            return len(self._branch[self._keymap['id']].lazyarray(
+                basketcache=BASKET_CACHE)[self._index])
 
     def __str__(self):
         return "Number of elements: {}".format(len(self._branch))
 
     def __repr__(self):
-        return "<{}[{}]: {} parsed elements>".format(self.__class__.__name__, self._mapper.name,
-                                                 len(self))
+        length = len(self)
+        return "<{}[{}]: {} element{}>".format(self.__class__.__name__,
+                                               self._mapper.name, length,
+                                               's' if length > 1 else '')