Skip to content
Snippets Groups Projects
Verified Commit 84e4fbbc authored by Tamas Gal's avatar Tamas Gal :speech_balloon:
Browse files

Deprecate old Summaryslice reader and introduce the new one

parent 8ad906d3
No related branches found
No related tags found
1 merge request!68Rewrite the online module
Pipeline #25153 failed
Stage: test
Stage: coverage
Stage: doc
Hide whitespace changes
Inline Side-by-side
km3io/online.py
+ 51
− 77
View file @ 84e4fbbc
import binascii
from collections import namedtuple
import os
import uproot
import uproot3
......@@ -22,36 +23,70 @@ RATE_FACTOR = np.log(MAXIMAL_RATE_HZ / MINIMAL_RATE_HZ) / 255
CHANNEL_BITS_TEMPLATE = np.zeros(31, dtype=bool)
SummarysliceChunk = namedtuple(field_names=["headers", "slices"], typename="SummarysliceChunk")
class SummarysliceReader:
"""
A reader for summaryslices which are loaded as chunks given by step_size.
To be used as an iterator (`for chunks in SummarysliceReader(...): ...`)
"""
TREE_ADDR = "KM3NET_SUMMARYSLICE/KM3NET_SUMMARYSLICE"
BRANCH_NAME = "vector<KM3NETDAQ::JDAQSummaryFrame>"
SUMMARYSLICES_BRANCH_NAME = "vector<KM3NETDAQ::JDAQSummaryFrame>"
HEADERS_BRANCH_NAME = "KM3NETDAQ::JDAQSummarysliceHeader"
SUMMARYSLICE_INTERPRETATION = uproot.interpretation.jagged.AsJagged(
uproot.interpretation.numerical.AsDtype(
[
("dom_id", ">i4"),
("dq_status", "u4"),
("hrv", "<u4"),
("fifo", "<u4"),
("status3", "u4"),
("status4", "u4"),
("dq_status", ">u4"),
("hrv", ">u4"),
("fifo", ">u4"),
("status3", ">u4"),
("status4", ">u4"),
]
+ [(f"ch{c}", "u1") for c in range(31)]
),
header_bytes=10,
)
HEADER_INTERPRETATION = uproot.interpretation.numerical.AsDtype(
[
(" cnt", "u4"),
(" vers", "u2"),
(" cnt2", "u4"),
(" vers2", "u2"),
(" cnt3", "u4"),
(" vers3", "u2"),
("detector_id", ">i4"),
("run", ">i4"),
("frame_index", ">i4"),
(" cnt4", "u4"),
(" vers4", "u2"),
("UTC_seconds", ">u4"),
("UTC_16nanosecondcycles", ">u4"),
]
)
def __init__(self, filename, step_size=1000):
self._filename = filename
def __init__(self, fobj, step_size=1000):
if isinstance(fobj, str):
self._fobj = uproot.open(fobj)
else:
self._fobj = fobj
self._step_size = step_size
self._fobj = uproot.open(filename)
self._branch = self._fobj[self.TREE_ADDR]
def _summaryslices_generator(self):
expressions = {self.BRANCH_NAME: self.SUMMARYSLICE_INTERPRETATION}
for summaryslices in self._branch.iterate(
expressions = {
self.SUMMARYSLICES_BRANCH_NAME: self.SUMMARYSLICE_INTERPRETATION,
self.HEADERS_BRANCH_NAME: self.HEADER_INTERPRETATION,
}
for chunk in self._branch.iterate(
expressions, step_size=self._step_size
):
yield summaryslices[self.BRANCH_NAME]
yield SummarysliceChunk(
chunk[self.HEADERS_BRANCH_NAME],
chunk[self.SUMMARYSLICES_BRANCH_NAME],
)
def __iter__(self):
self._summaryslices = self._summaryslices_generator()
......@@ -63,6 +98,9 @@ class SummarysliceReader:
def __len__(self):
return int(np.ceil(self._branch.num_entries / self._step_size))
def __repr__(self):
return f"<SummarysliceReader {self._branch.num_entries} summaryslices, step_size={self._step_size} ({len(self)} chunks)>"
@nb.vectorize(
[nb.int32(nb.int8), nb.int32(nb.int16), nb.int32(nb.int32), nb.int32(nb.int64)]
......@@ -224,74 +262,10 @@ class OnlineReader:
@property
def summaryslices(self):
if self._summaryslices is None:
self._summaryslices = SummarySlices(self._fobj)
self._summaryslices = SummarysliceReader(uproot.open(self._filename)) # TODO: remove when using uproot4
return self._summaryslices
class SummarySlices:
"""A wrapper for summary slices"""
def __init__(self, fobj):
self._fobj = fobj
self._slices = None
self._headers = None
self._rates = None
self._ch_selector = ["ch{}".format(c) for c in range(31)]
@property
def headers(self):
if self._headers is None:
self._headers = self._read_headers()
return self._headers
@property
def slices(self):
if self._slices is None:
self._slices = self._read_summaryslices()
return self._slices
@property
def rates(self):
if self._rates is None:
self._rates = self.slices[["dom_id"] + self._ch_selector]
return self._rates
def _read_summaryslices(self):
"""Reads a lazyarray of summary slices"""
tree = self._fobj[b"KM3NET_SUMMARYSLICE"][b"KM3NET_SUMMARYSLICE"]
return tree[b"vector<KM3NETDAQ::JDAQSummaryFrame>"].lazyarray(
uproot3.asjagged(
uproot3.astable(
uproot3.asdtype(
[
("dom_id", "i4"),
("dq_status", "u4"),
("hrv", "u4"),
("fifo", "u4"),
("status3", "u4"),
("status4", "u4"),
]
+ [(c, "u1") for c in self._ch_selector]
)
),
skipbytes=10,
),
basketcache=uproot3.cache.ThreadSafeArrayCache(
SUMMARYSLICE_FRAME_BASKET_CACHE_SIZE
),
)
def _read_headers(self):
"""Reads a lazyarray of summary slice headers"""
tree = self._fobj[b"KM3NET_SUMMARYSLICE"][b"KM3NET_SUMMARYSLICE"]
return tree[b"KM3NETDAQ::JDAQSummarysliceHeader"].lazyarray(
uproot3.interpret(tree[b"KM3NETDAQ::JDAQSummarysliceHeader"], cntvers=True)
)
def __str__(self):
return "Number of summaryslices: {}".format(len(self.headers))
class Timeslices:
"""A simple wrapper for timeslices"""
......
tests/test_online.py
+ 18
− 16
View file @ 84e4fbbc
......@@ -175,7 +175,9 @@ class TestTimeslice(unittest.TestCase):
class TestSummaryslices(unittest.TestCase):
def setUp(self):
self.ss = OnlineReader(ONLINE_FILE).summaryslices
for chunk in OnlineReader(ONLINE_FILE).summaryslices:
self.ss = chunk
break
def test_headers(self):
assert 3 == len(self.ss.headers)
......@@ -187,9 +189,6 @@ class TestSummaryslices(unittest.TestCase):
def test_slices(self):
assert 3 == len(self.ss.slices)
def test_rates(self):
assert 3 == len(self.ss.rates)
def test_fifo(self):
s = self.ss.slices[0]
dct_fifo_stat = {
......@@ -700,7 +699,10 @@ class TestGetChannelFlags_Issue59(unittest.TestCase):
r = OnlineReader(
data_path("online/KM3NeT_00000049_00008456.summaryslice-167941.root")
)
summaryslice = r.summaryslices.slices[0]
for chunks in r.summaryslices:
summaryslice = chunks.slices[0]
break
for ours, ref in zip(summaryslice, ref_entries):
assert ours.dom_id == to_num(ref.dom_id)
......@@ -766,16 +768,16 @@ class TestSummarysliceReader(unittest.TestCase):
for ss in sr:
self.assertListEqual(
ss[0].dom_id[:3].to_list(), [806451572, 806455814, 806465101]
ss.slices[0].dom_id[:3].to_list(), [806451572, 806455814, 806465101]
)
self.assertListEqual(
ss[0].dom_id[-3:].to_list(), [809526097, 809544058, 809544061]
ss.slices[0].dom_id[-3:].to_list(), [809526097, 809544058, 809544061]
)
assert len(ss) == 3
assert len(ss[0]) == 64
assert len(ss[1]) == 66
assert len(ss[2]) == 68
self.assertListEqual(ss[0].ch5[:3].to_list(), [75, 62, 55])
assert len(ss.slices) == 3
assert len(ss.slices[0]) == 64
assert len(ss.slices[1]) == 66
assert len(ss.slices[2]) == 68
self.assertListEqual(ss.slices[0].ch5[:3].to_list(), [75, 62, 55])
sr = SummarysliceReader(data_path("online/km3net_online.root"), step_size=1)
......@@ -784,7 +786,7 @@ class TestSummarysliceReader(unittest.TestCase):
for idx, ss in enumerate(sr):
# self.assertListEqual(ss[0].dom_id[:3].to_list(), [806451572, 806455814, 806465101])
# self.assertListEqual(ss[0].dom_id[-3:].to_list(), [809526097, 809544058, 809544061])
assert len(ss) == 1
assert len(ss[0]) == lengths[idx]
assert len(ss[0].dom_id) == lengths[idx]
assert len(ss[0].ch3) == lengths[idx]
assert len(ss.slices) == 1
assert len(ss.slices[0]) == lengths[idx]
assert len(ss.slices[0].dom_id) == lengths[idx]
assert len(ss.slices[0].ch3) == lengths[idx]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment