diff --git a/README.md b/README.md
index de540f579429cb6f13e42c8b56a2789bf1a65478..a2061afdcc2b2bfe737679d8f216a00174b54b78 100644
--- a/README.md
+++ b/README.md
@@ -32,17 +32,47 @@ export TAGS_TO_MIRROR="IO_EVT, IO_SUM, IO_TSL, IO_TSL0, IO_TSL1, IO_TSL2, IO_TSS
```
-After that, use the following command to start the ``supervisor``:
+After that, use the following command to start the ``supervisor``, which
+you only need to do once:
source setenv.sh
make start
-To see the status of the processes, use ``supervisorctl status``
+From now on ``supervisorctl`` is the tool to communicate with the monitoring
+system. To see the status of the processes, use ``supervisorctl status``,
+which will show each process one by one (make sure you call it in the
+folder where you launched it):
+```
+$ supervisorctl status
+ligiers:ligiermirror RUNNING pid 611, uptime 1 day, 7:55:09
+ligiers:monitoring_ligier RUNNING pid 610, uptime 1 day, 7:55:09
+logging:msg_dumper RUNNING pid 7466, uptime 1 day, 7:28:00
+logging:weblog RUNNING pid 7465, uptime 1 day, 7:28:00
+monitoring_process:ahrs_calibration RUNNING pid 19612, uptime 1 day, 1:20:32
+monitoring_process:dom_activity RUNNING pid 626, uptime 1 day, 7:55:09
+monitoring_process:dom_rates RUNNING pid 631, uptime 1 day, 7:55:09
+monitoring_process:pmt_hrv RUNNING pid 633, uptime 1 day, 7:55:09
+monitoring_process:pmt_rates RUNNING pid 632, uptime 1 day, 7:55:09
+monitoring_process:rttc RUNNING pid 9717, uptime 10:55:53
+monitoring_process:trigger_rates RUNNING pid 637, uptime 1 day, 7:55:09
+monitoring_process:triggermap RUNNING pid 638, uptime 1 day, 7:55:09
+monitoring_process:ztplot RUNNING pid 7802, uptime 1 day, 7:26:13
+webserver RUNNING pid 29494, uptime 1 day, 0:34:23
+```
+
+The processes are grouped accordingly (ligier, monitoring_process etc.) and
+automaticallly started in the right order.
You can stop and start individual services using ``supervisorctl stop
group:process_name`` and ``supervisorctl start group:process_name``
+Since the system knows the order, you can safely ``restart all`` or just
+a group of processes. Use the ``supervisorctl help`` to find out more and
+``supervisorctl help COMMAND`` to get a detailed description of the
+corresponding command.
+
+
## Configuration file
A file called `pipeline.toml` can be placed into the root folder of the
diff --git a/app/routes.py b/app/routes.py
index 11234379ae61ddbadc2c396ddf256aa85dab444e..03ead0b0d36ea3d297bb5693a09ca788a3b0d4ec 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -1,4 +1,5 @@
-from os.path import join, exists
+from glob import glob
+from os.path import basename, join, exists, splitext
from functools import wraps
import toml
from flask import render_template, send_from_directory, request, Response
@@ -13,7 +14,7 @@ app.config['FREEZER_DESTINATION'] = '../km3web'
PLOTS = [['dom_activity', 'dom_rates'], ['pmt_rates', 'pmt_hrv'],
['trigger_rates'], ['ztplot', 'triggermap']]
-AHRS_PLOTS = [['yaw_calib'], ['pitch_calib'], ['roll_calib']]
+AHRS_PLOTS = ['yaw_calib_du*', 'pitch_calib_du*', 'roll_calib_du*']
TRIGGER_PLOTS = [['trigger_rates'], ['trigger_rates_lin']]
K40_PLOTS = [['intradom'], ['angular_k40rate_distribution']]
RTTC_PLOTS = [['rttc']]
@@ -21,7 +22,7 @@ RECO_PLOTS = [['track_reco', 'ztplot_roy'], ['time_residuals']]
COMPACT_PLOTS = [['dom_activity', 'dom_rates', 'pmt_rates', 'pmt_hrv'],
['trigger_rates', 'trigger_rates_lin'],
['ztplot', 'ztplot_roy', 'triggermap']]
-SN_PLOTS = [['sn_bg_distribution']]
+SN_PLOTS = [['sn_bg_histogram', 'sn_pk_history']]
if exists(CONFIG_PATH):
config = toml.load(CONFIG_PATH)
@@ -31,6 +32,21 @@ if exists(CONFIG_PATH):
PASSWORD = config["WebServer"]["password"]
+def expand_wildcards(plot_layout):
+ """Replace wildcard entries with list of files"""
+ plots = []
+ for row in plot_layout:
+ if not isinstance(row, list) and '*' in row:
+ plots.append(
+ sorted([
+ splitext(basename(p))[0]
+ for p in glob(join(app.root_path, PLOTS_PATH, row))
+ ]))
+ else:
+ plots.append(row)
+ return plots
+
+
def check_auth(username, password):
"""This function is called to check if a username /
password combination is valid.
@@ -76,29 +92,31 @@ def add_header(r):
@app.route('/index.html')
@requires_auth
def index():
- return render_template('plots.html', plots=PLOTS)
+ return render_template('plots.html', plots=expand_wildcards(PLOTS))
@app.route('/ahrs.html')
@requires_auth
def ahrs():
- return render_template('plots.html', plots=AHRS_PLOTS)
+ return render_template('plots.html', plots=expand_wildcards(AHRS_PLOTS))
@app.route('/reco.html')
@requires_auth
def reco():
- return render_template('plots.html', plots=RECO_PLOTS)
+ return render_template('plots.html', plots=expand_wildcards(RECO_PLOTS))
+
@app.route('/sn.html')
@requires_auth
def supernova():
- return render_template('plots.html', plots=SN_PLOTS)
+ return render_template('plots.html', plots=expand_wildcards(SN_PLOTS))
+
@app.route('/compact.html')
@requires_auth
def compact():
- return render_template('plots.html', plots=COMPACT_PLOTS)
+ return render_template('plots.html', plots=expand_wildcards(COMPACT_PLOTS))
@app.route('/rttc.html')
@@ -106,7 +124,7 @@ def compact():
def rttc():
return render_template(
'plots.html',
- plots=RTTC_PLOTS,
+ plots=expand_wildcards(RTTC_PLOTS),
info=
"Cable Round Trip Time calculated from realtime data provided by the "
"Detector Manager. The red lines shows the median and the STD "
@@ -119,7 +137,7 @@ def rttc():
def k40():
return render_template(
'plots.html',
- plots=K40_PLOTS,
+ plots=expand_wildcards(K40_PLOTS),
info="The first plot shows the intra-DOM calibration. "
"y-axis: delta_t [ns], x-axis: cosine of angles. "
"The second plot the angular distribution of K40 rates. "
@@ -130,7 +148,7 @@ def k40():
@app.route('/trigger.html')
@requires_auth
def trigger():
- return render_template('plots.html', plots=TRIGGER_PLOTS)
+ return render_template('plots.html', plots=expand_wildcards(TRIGGER_PLOTS))
@app.route('/plots/<path:filename>')
diff --git a/scripts/ahrs_calibration.py b/scripts/ahrs_calibration.py
index d5db82c16da907c693fec202e1369f8a0b99058d..61d3a09c8d7de8ec88898968c92ae45cbbf1a194 100755
--- a/scripts/ahrs_calibration.py
+++ b/scripts/ahrs_calibration.py
@@ -44,21 +44,29 @@ class CalibrateAHRS(kp.Module):
self.plots_path = self.require('plots_path')
det_id = self.require('det_id')
self.detector = kp.hardware.Detector(det_id=det_id)
- self.du = self.get('du', default=1)
+ self.dus = set()
self.clbmap = kp.db.CLBMap(det_oid=det_id)
self.cuckoo = kp.time.Cuckoo(60, self.create_plot)
self.cuckoo_log = kp.time.Cuckoo(10, print)
+
self.data = {}
- queue_size = 50000
- for ahrs_param in ('yaw', 'pitch', 'roll'):
- self.data[ahrs_param] = defaultdict(
- partial(deque, maxlen=queue_size))
- self.times = defaultdict(partial(deque, maxlen=queue_size))
+ self.queue_size = 50000
+
self.lock = threading.Lock()
self.index = 0
+ def _register_du(self, du):
+ """Create data cache for DU"""
+ self.data[du] = {}
+ for ahrs_param in ('yaw', 'pitch', 'roll'):
+ self.data[du][ahrs_param] = defaultdict(
+ partial(deque, maxlen=self.queue_size))
+ self.data[du]['times'] = defaultdict(
+ partial(deque, maxlen=self.queue_size))
+ self.dus.add(du)
+
def process(self, blob):
self.index += 1
if self.index % 29 != 0:
@@ -71,23 +79,24 @@ class CalibrateAHRS(kp.Module):
self.log.info("Skipping base CLB")
return blob
- if clb.du != self.du:
- return blob
-
yaw = tmch_data.yaw
calib = get_latest_ahrs_calibration(clb.upi, max_version=4)
if calib is None:
+ self.log.warning("No calibration found for CLB UPI '%s'", clb.upi)
return blob
+ du = clb.du
+ if du not in self.dus:
+ self._register_du(du)
cyaw, cpitch, croll = fit_ahrs(tmch_data.A, tmch_data.H, *calib)
self.cuckoo_log("DU{}-DOM{} (random pick): calibrated yaw={}".format(
clb.du, clb.floor, cyaw))
with self.lock:
- self.data['yaw'][clb.floor].append(cyaw)
- self.data['pitch'][clb.floor].append(cpitch)
- self.data['roll'][clb.floor].append(croll)
- self.times[clb.floor].append(now)
+ self.data[du]['yaw'][clb.floor].append(cyaw)
+ self.data[du]['pitch'][clb.floor].append(cpitch)
+ self.data[du]['roll'][clb.floor].append(croll)
+ self.data[du]['times'][clb.floor].append(now)
self.cuckoo.msg()
return blob
@@ -96,30 +105,33 @@ class CalibrateAHRS(kp.Module):
print(self.__class__.__name__ + ": updating plot.")
# xfmt = md.DateFormatter('%Y-%m-%d %H:%M')
xfmt = md.DateFormatter('%H:%M')
- for ahrs_param in self.data.keys():
- fig, ax = plt.subplots(figsize=(16, 6))
- sns.set_palette("husl", 18)
- ax.set_title("AHRS {} Calibration on DU{}\n{}".format(
- ahrs_param, self.du, datetime.utcnow()))
- ax.set_xlabel("UTC time")
- ax.xaxis.set_major_formatter(xfmt)
- ax.set_ylabel(ahrs_param)
- with self.lock:
- for floor in sorted(self.data[ahrs_param].keys()):
- ax.plot(
- self.times[floor],
- self.data[ahrs_param][floor],
- marker='.',
- linestyle='none',
- label="Floor {}".format(floor))
- lgd = plt.legend(
- bbox_to_anchor=(1.005, 1), loc=2, borderaxespad=0.)
- fig.tight_layout()
- plt.savefig(
- os.path.join(self.plots_path, ahrs_param + '_calib.png'),
- bbox_extra_artists=(lgd, ),
- bbox_inches='tight')
- plt.close('all')
+ for du in self.dus:
+ data = self.data[du]
+ for ahrs_param in data.keys():
+ fig, ax = plt.subplots(figsize=(16, 6))
+ sns.set_palette("husl", 18)
+ ax.set_title("AHRS {} Calibration on DU{}\n{}".format(
+ ahrs_param, du, datetime.utcnow()))
+ ax.set_xlabel("UTC time")
+ ax.xaxis.set_major_formatter(xfmt)
+ ax.set_ylabel(ahrs_param)
+ with self.lock:
+ for floor in sorted(data[ahrs_param].keys()):
+ ax.plot(data['times'][floor],
+ data[ahrs_param][floor],
+ marker='.',
+ linestyle='none',
+ label="Floor {}".format(floor))
+ lgd = plt.legend(bbox_to_anchor=(1.005, 1),
+ loc=2,
+ borderaxespad=0.)
+ fig.tight_layout()
+ plt.savefig(os.path.join(
+ self.plots_path,
+ ahrs_param + '_calib_du{}.png'.format(du)),
+ bbox_extra_artists=(lgd, ),
+ bbox_inches='tight')
+ plt.close('all')
def main():
@@ -132,13 +144,12 @@ def main():
ligier_port = int(args['-p'])
pipe = kp.Pipeline()
- pipe.attach(
- kp.io.ch.CHPump,
- host=ligier_ip,
- port=ligier_port,
- tags='IO_MONIT',
- timeout=60 * 60 * 24 * 7,
- max_queue=2000)
+ pipe.attach(kp.io.ch.CHPump,
+ host=ligier_ip,
+ port=ligier_port,
+ tags='IO_MONIT',
+ timeout=60 * 60 * 24 * 7,
+ max_queue=2000)
pipe.attach(kp.io.daq.DAQProcessor)
pipe.attach(CalibrateAHRS, det_id=det_id, plots_path=plots_path)
pipe.drain()