diff --git a/docs/api.rst b/docs/api.rst
deleted file mode 100644
index b44f8f30c5544a3b2af63ec52f9f801b348c0486..0000000000000000000000000000000000000000
--- a/docs/api.rst
+++ /dev/null
@@ -1,69 +0,0 @@
-API Reference
-=============
-
-.. contents:: :local:
-
-OrcaSong: Main Framework
-------------------------
-
-``orcasong.data_to_images``: Main code
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: orcasong.data_to_images
- :no-members:
- :no-inherited-members:
-
-.. currentmodule:: orcasong.data_to_images
-
-.. autosummary::
- :toctree: api
-
- parse_input
- parser_check_input
- make_output_dirs
- calculate_bin_edges
- calculate_bin_edges_test
- get_file_particle_type
- EventSkipper
- skip_event
- data_to_images
- main
-
-
-``orcasong.file_to_hits``: Extracting event information
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: orcasong.file_to_hits
- :no-members:
- :no-inherited-members:
-
-.. currentmodule:: orcasong.file_to_hits
-
-.. autosummary::
- :toctree: api
-
- get_primary_track_index
- get_time_residual_nu_interaction_mean_triggered_hits
- get_hits
- get_tracks
- EventDataExtractor
-
-
-``orcasong.hits_to_histograms``: Making images based on the event info
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: orcasong.hits_to_histograms
- :no-members:
- :no-inherited-members:
-
-.. currentmodule:: orcasong.hits_to_histograms
-
-.. autosummary::
- :toctree: api
-
- get_time_parameters
- compute_4d_to_2d_histograms
- convert_2d_numpy_hists_to_pdf_image
- compute_4d_to_3d_histograms
- compute_4d_to_4d_histograms
- HistogramMaker
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index f08f846028c69e245eff83efe58bc81089bc40f1..bdd9b9a6d63b030c8a08807b4cb9692807dfc8eb 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -46,11 +46,24 @@ extensions = [
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
'sphinx.ext.autosummary',
+ 'autoapi.extension',
'numpydoc',
]
autosummary_generate = True
+# Document Python Code
+autoapi_type = 'python'
+autoapi_dirs = ['../orcasong']
+autoapi_options = [
+ 'members', 'undoc-members'
+ # , 'private-members', 'special-members'
+]
+autoapi_ignore = [
+ "*/tests/*", "*test_*.py", "*/doc/conf.py", "*/pydataskel*", "*/style/*"
+]
+autoapi_include_summaries = True
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 006c39ef70834d26751bfbeded7274673b49bb64..5b3cb1fdaae9b2b6350187ecb1e7fea524b883f4 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -149,106 +149,85 @@ Currently, only the 115l ORCA 2016 detx file is available.
At this point, you're finally ready to use OrcaSong.
OrcaSong can be called from every directory by using the :code:`make_nn_images` command::
- ~/$: make_nn_images testfile.h5 geofile.detx
+ ~/$: make_nn_images testfile.h5 geofile.detx configfile.toml
+
+OrcaSong will then generate a hdf5 file with images that will be put in a "Results" folder at the path that
+you've specified in the configfile current path.
+Please checkout the default_config.toml file in the orcasong folder of the OrcaSong repo in order to get an idea about
+the structure of the config files.
+
+All available configuration options of OrcaSong can be found in /orcasong/default_config::
+
+ --- Documentation for every config parameter that is available ---
+
+ None arguments should be written as string: 'None'
+
+ Parameters
+ ----------
+ output_dirpath : str
+ Full path to the directory, where the orcasong output should be stored.
+ chunksize : int
+ Chunksize (along axis_0) that is used for saving the OrcaSong output to a .h5 file.
+ complib : str
+ Compression library that is used for saving the OrcaSong output to a .h5 file.
+ All PyTables compression filters are available, e.g. 'zlib', 'lzf', 'blosc', ... .
+ complevel : int
+ Compression level for the compression filter that is used for saving the OrcaSong output to a .h5 file.
+ n_bins : tuple of int
+ Declares the number of bins that should be used for each dimension, e.g. (x,y,z,t).
+ The option should be written as string, e.g. '11,13,18,60'.
+ det_geo : str
+ Declares what detector geometry should be used for the binning. E.g. 'Orca_115l_23m_h_9m_v'.
+ do2d : bool
+ Declares if 2D histograms, 'images', should be created.
+ do2d_plots : bool
+ Declares if pdf visualizations of the 2D histograms should be created, cannot be called if do2d=False.
+ do2d_plots_n: int
+ After how many events the event loop will be stopped (making the 2d plots in do2d_plots takes long time).
+ do3d : bool
+ Declares if 3D histograms should be created.
+ do4d : bool
+ Declares if 4D histograms should be created.
+ do4d_mode : str
+ If do4d is True, what should be used as the 4th dim after xyz.
+ Currently, only 'time' and 'channel_id' are available.
+ prod_ident : int
+ Optional int identifier for the used mc production.
+ This is e.g. useful, if you use events from two different mc productions, e.g. the 1-5GeV & 3-100GeV Orca 2016 MC.
+ In this case, the events are not fully distinguishable with only the run_id and the event_id!
+ In order to keep a separation, an integer can be set in the event_track for all events, such that they stay distinguishable.
+ timecut_mode : str
+ Defines what timecut should be used in hits_to_histograms.py.
+ Currently available:
+ 'timeslice_relative': Cuts out the central 30% of the snapshot. The value of timecut_timespan doesn't matter in this case.
+ 'trigger_cluster': Cuts based on the mean of the triggered hits.
+ 'None': No timecut. The value of timecut_timespan doesn't matter in this case.
+ timecut_timespan : str/None
+ Defines what timespan should be used if a timecut is applied. Only relevant for timecut_mode = 'trigger_cluster'.
+ Currently available:
+ 'all': [-350ns, 850ns] -> 20ns / bin (if e.g. 60 timebins)
+ 'tight-0': [-450ns, 500ns] -> 15.8ns / bin (if e.g. 60 timebins)
+ 'tight-1': [-250ns, 500ns] -> 12.5ns / bin (if e.g. 60 timebins)
+ 'tight-2': [-150ns, 200ns] -> 5.8ns / bin (if e.g. 60 timebins)
+ do_mc_hits : bool
+ Declares if hits (False, mc_hits + BG) or mc_hits (True) should be processed.
+ data_cut_triggered : bool
+ Cuts away hits that haven't been triggered.
+ data_cut_e_low : float
+ Cuts away events that have an energy lower than data_cut_e_low.
+ data_cut_e_high : float
+ Cuts away events that have an energy higher than data_cut_e_high.
+ data_cut_throw_away : float
+ Cuts away random events with a certain probability (1: 100%, 0: 0%).
+ flush_freq : int
+ After how many events the accumulated output should be flushed to the harddisk.
+ A larger value leads to a faster orcasong execution, but it increases the RAM usage as well.
+
+ --- Documentation for every config parameter that is available ---
-OrcaSong will then generate a hdf5 file with images that will be put in a "Results" folder at your current path.
-The configuration options of OrcaSong can be found by calling the help::
- ~/$: make_nn_images -h
- Main OrcaSong code which takes raw simulated .h5 files and the corresponding .detx detector file as input in
- order to generate 2D/3D/4D histograms ('images') that can be used for CNNs.
- First argument: KM3NeT hdf5 simfile at JTE level.
- Second argument: a .detx file that is associated with the hdf5 file.
-
- The input file can be calibrated or not (e.g. contains pos_xyz of the hits) and the OrcaSong output is written
- to the current folder by default (otherwise use --o option).
- Makes only 4D histograms ('images') by default.
-
- Usage:
- data_to_images.py [options] FILENAME DETXFILE
- data_to_images.py (-h | --help)
-
- Options:
- -h --help Show this screen.
-
- -c CONFIGFILE Load all options from a config file (.toml format).
-
- --o OUTPUTPATH Path for the directory, where the OrcaSong output should be stored. [default: ./]
-
- --chunksize CHUNKSIZE Chunksize (axis_0) that should be used for the hdf5 output of OrcaSong. [default: 32]
-
- --complib COMPLIB Compression library that should be used for the OrcaSong output.
- All PyTables compression filters are available. [default: zlib]
-
- --complevel COMPLEVEL Compression level that should be used for the OrcaSong output. [default: 1]
-
- --n_bins N_BINS Number of bins that are used in the image making for each dimension, e.g. (x,y,z,t).
- [default: 11,13,18,60]
-
- --det_geo DET_GEO Which detector geometry to use for the binning, e.g. 'Orca_115l_23m_h_9m_v'.
- [default: Orca_115l_23m_h_9m_v]
-
- --do2d If 2D histograms, 'images', should be created.
-
- --do2d_plots If 2D pdf plot visualizations of the 2D histograms should be created, cannot be called if do2d=False.
-
- --do2d_plots_n N For how many events the 2D plot visualizations should be made.
- OrcaSong will exit after reaching N events. [default: 10]
-
- --do3d If 3D histograms, 'images', should be created.
-
- --dont_do4d If 4D histograms, 'images', should NOT be created.
-
- --do4d_mode MODE What dimension should be used in the 4D histograms as the 4th dim.
- Available: 'time', 'channel_id'. [default: time]
-
- --timecut_mode MODE Defines what timecut mode should be used in hits_to_histograms.py.
- At the moment, these cuts are only optimized for ORCA 115l neutrino events!
- Currently available:
- 'timeslice_relative': Cuts out the central 30% of the snapshot.
- 'trigger_cluster': Cuts based on the mean of the triggered hits.
- The timespan for this cut can be chosen in --timecut_timespan.
- 'None': No timecut.
- [default: trigger_cluster]
-
- --timecut_timespan TIMESPAN Only used with timecut_mode 'trigger_cluster'.
- Defines the timespan of the trigger_cluster cut.
- Currently available:
- 'all': [-350ns, 850ns] -> 20ns / bin (60 bins)
- 'tight-1': [-250ns, 500ns] -> 12.5ns / bin
- 'tight-2': [-150ns, 200ns] -> 5.8ns / bin
- [default: tight-1]
-
- --do_mc_hits If only the mc_hits (no BG hits!) should be used for the image processing.
-
- --data_cut_triggered If non-triggered hits should be thrown away for the images.
-
- --data_cut_e_low E_LOW Cut events that are lower than the specified threshold value in GeV.
-
- --data_cut_e_high E_HIGH Cut events that are higher than the specified threshold value in GeV.
-
- --data_cut_throw_away FRACTION Throw away a random fraction (percentage) of events. [default: 0.00]
-
- --prod_ident PROD_IDENT Optional int identifier for the used mc production.
- This is useful, if you use events from two different mc productions,
- e.g. the 1-5GeV & 3-100GeV Orca 2016 MC. The prod_ident int will be saved in
- the 'y' dataset of the output file of OrcaSong. [default: 1]
-
-
-Alternatively, they can also be found in the docs of the :code:`data_to_images()` function:
-
-.. currentmodule:: orcasong.data_to_images
-.. autosummary::
- data_to_images
-
-Other than parsing every information to orcasong via the console, you can also load a .toml config file::
-
- ~/$: make_nn_images -c config.toml testfile.h5 geofile.detx
-
-Please checkout the config.toml file in the main folder of the OrcaSong repo in order to get an idea about
-the structure of the config file.
If anything is still unclear after this introduction just tell me in the deep_learning channel on chat.km3net.de or
write me an email at michael.m.moser@fau.de, such that I can improve this guide!
diff --git a/docs/index.rst b/docs/index.rst
index 69c2eca2127bde5ebf669f33d6924b514c36bed7..980703946f9b23c0fc2bb0dc6af082b4a6ad91c1 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -27,8 +27,7 @@ These event 'images' are required for some Deep Learning machine learning algori
As of now, only ORCA detector simulations are supported, but ARCA geometries can be easily implemented as well.
-The main code for generating the images is located in orcanet/data_to_images.py.
-If the simulated hdf5 files are not calibrated yet, you need to specify the directory of a .detx file in 'data_to_images.py'.
+The main code for generating the images is located in orcanet/make_nn_images.py.
As of now, the documentation contains a small introduction to get started and and a complete API documentation.
Please feel free to contact me or just open an issue on Gitlab / Github if you have any suggestions.
@@ -38,7 +37,6 @@ Please feel free to contact me or just open an issue on Gitlab / Github if you h
:caption: Contents:
getting_started
- api
Indices and tables
diff --git a/orcasong/__version__.py b/orcasong/__version__.py
index 45827bfcdbba920bf1b00332951bd2a7761074a6..6879d8da2ed22c3bb5bdaa208c53f071e660bf1d 100644
--- a/orcasong/__version__.py
+++ b/orcasong/__version__.py
@@ -18,5 +18,8 @@ version = 'unknown version'
try:
version = get_version(root='..', relative_to=__file__)
except LookupError:
- with open(join(realpath(dirname(__file__)), "version.txt"), 'r') as fobj:
- version = fobj.read()
+ try:
+ with open(join(realpath(dirname(__file__)), "version.txt"), 'r') as fobj:
+ version = fobj.read()
+ except IOError:
+ pass
diff --git a/orcasong/default_config.toml b/orcasong/default_config.toml
new file mode 100644
index 0000000000000000000000000000000000000000..a995541d6b8849c9b4bda736dd4e772d2f2519f8
--- /dev/null
+++ b/orcasong/default_config.toml
@@ -0,0 +1,99 @@
+# A config file for OrcaSong with a list of all configuration options.
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+# IMPORTANT: Don't remove any config lines, only modify them!
+
+#--- Documentation for every config parameter that is available ---#
+#
+# None arguments should be written as string: 'None'
+#
+# Parameters
+# ----------
+# output_dirpath : str
+# Full path to the directory, where the orcasong output should be stored.
+# chunksize : int
+# Chunksize (along axis_0) that is used for saving the OrcaSong output to a .h5 file.
+# complib : str
+# Compression library that is used for saving the OrcaSong output to a .h5 file.
+# All PyTables compression filters are available, e.g. 'zlib', 'lzf', 'blosc', ... .
+# complevel : int
+# Compression level for the compression filter that is used for saving the OrcaSong output to a .h5 file.
+# n_bins : tuple of int
+# Declares the number of bins that should be used for each dimension, e.g. (x,y,z,t).
+# The option should be written as string, e.g. '11,13,18,60'.
+# det_geo : str
+# Declares what detector geometry should be used for the binning. E.g. 'Orca_115l_23m_h_9m_v'.
+# do2d : bool
+# Declares if 2D histograms, 'images', should be created.
+# do2d_plots : bool
+# Declares if pdf visualizations of the 2D histograms should be created, cannot be called if do2d=False.
+# do2d_plots_n: int
+# After how many events the event loop will be stopped (making the 2d plots in do2d_plots takes long time).
+# do3d : bool
+# Declares if 3D histograms should be created.
+# do4d : bool
+# Declares if 4D histograms should be created.
+# do4d_mode : str
+# If do4d is True, what should be used as the 4th dim after xyz.
+# Currently, only 'time' and 'channel_id' are available.
+# prod_ident : int
+# Optional int identifier for the used mc production.
+# This is e.g. useful, if you use events from two different mc productions, e.g. the 1-5GeV & 3-100GeV Orca 2016 MC.
+# In this case, the events are not fully distinguishable with only the run_id and the event_id!
+# In order to keep a separation, an integer can be set in the event_track for all events, such that they stay distinguishable.
+# timecut_mode : str
+# Defines what timecut should be used in hits_to_histograms.py.
+# Currently available:
+# 'timeslice_relative': Cuts out the central 30% of the snapshot. The value of timecut_timespan doesn't matter in this case.
+# 'trigger_cluster': Cuts based on the mean of the triggered hits.
+# 'None': No timecut. The value of timecut_timespan doesn't matter in this case.
+# timecut_timespan : str/None
+# Defines what timespan should be used if a timecut is applied. Only relevant for timecut_mode = 'trigger_cluster'.
+# Currently available:
+# 'all': [-350ns, 850ns] -> 20ns / bin (if e.g. 60 timebins)
+# 'tight-0': [-450ns, 500ns] -> 15.8ns / bin (if e.g. 60 timebins)
+# 'tight-1': [-250ns, 500ns] -> 12.5ns / bin (if e.g. 60 timebins)
+# 'tight-2': [-150ns, 200ns] -> 5.8ns / bin (if e.g. 60 timebins)
+# do_mc_hits : bool
+# Declares if hits (False, mc_hits + BG) or mc_hits (True) should be processed.
+# data_cut_triggered : bool
+# Cuts away hits that haven't been triggered.
+# data_cut_e_low : float
+# Cuts away events that have an energy lower than data_cut_e_low.
+# data_cut_e_high : float
+# Cuts away events that have an energy higher than data_cut_e_high.
+# data_cut_throw_away : float
+# Cuts away random events with a certain probability (1: 100%, 0: 0%).
+# flush_freq : int
+# After how many events the accumulated output should be flushed to the harddisk.
+# A larger value leads to a faster orcasong execution, but it increases the RAM usage as well.
+#
+#--- Documentation for every config parameter that is available ---#
+
+#--- All available config options with some default values ---#
+
+output_dirpath = './'
+chunksize = 32
+complib = 'zlib'
+complevel = 1
+n_bins = '11,13,18,60'
+det_geo = 'Orca_115l_23m_h_9m_v'
+do2d = false
+do2d_plots = false
+do2d_plots_n = 10
+do3d = false
+do4d = true
+do4d_mode = 'time'
+timecut_mode = 'trigger_cluster'
+timecut_timespan = 'tight_1'
+do_mc_hits = false
+data_cut_triggered = false
+data_cut_e_low = 'None'
+data_cut_e_high = 'None'
+data_cut_throw_away = 0.00
+prod_ident = 1
+flush_freq = 1000
+
+#--- All available config options with some default values ---#
+
+
diff --git a/orcasong/file_to_hits.py b/orcasong/file_to_hits.py
index 2044b54c12ef63ac0b95b5eac353eac0548eb14d..449ebc33324548fabd736e32eb84722c7d827d10 100644
--- a/orcasong/file_to_hits.py
+++ b/orcasong/file_to_hits.py
@@ -148,7 +148,7 @@ def get_tracks(event_blob, file_particle_type, event_hits, prod_ident):
if file_particle_type == 'undefined': # currently used with random_noise files
run_id = event_blob['EventInfo'].run_id
else:
- raise InputError('The run_id could not be read from the EventInfo or the Header, '
+ raise ValueError('The run_id could not be read from the EventInfo or the Header, '
'please check the source code in get_tracks().')
## collect all event_track information, dependent on file_particle_type
diff --git a/orcasong/data_to_images.py b/orcasong/make_nn_images.py
similarity index 59%
rename from orcasong/data_to_images.py
rename to orcasong/make_nn_images.py
index bf29f4351e893309328fd69e9bf058e30a2f5fea..0a405a945f255180316e72e0d5a77b8dc6b08f38 100644
--- a/orcasong/data_to_images.py
+++ b/orcasong/make_nn_images.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# coding=utf-8
-# Filename: data_to_images.py
+# Filename: make_nn_images.py
"""
Main OrcaSong code which takes raw simulated .h5 files and the corresponding .detx detector file as input in
order to generate 2D/3D/4D histograms ('images') that can be used for CNNs.
@@ -8,79 +8,23 @@ order to generate 2D/3D/4D histograms ('images') that can be used for CNNs.
First argument: KM3NeT hdf5 simfile at JTE level.
Second argument: a .detx file that is associated with the hdf5 file.
-The input file can be calibrated or not (e.g. contains pos_xyz of the hits) and the OrcaSong output is written
-to the current folder by default (otherwise use --o option).
-Makes only 4D histograms ('images') by default.
+The input file can be calibrated or not (e.g. contains pos_xyz of the hits).
Usage:
- data_to_images.py [options] FILENAME DETXFILE
- data_to_images.py (-h | --help)
+ make_nn_images.py SIMFILE DETXFILE CONFIGFILE
+ make_nn_images.py (-h | --help)
-Options:
- -h --help Show this screen.
-
- -c CONFIGFILE Load all options from a config file (.toml format).
-
- --o OUTPUTPATH Path for the directory, where the OrcaSong output should be stored. [default: ./]
-
- --chunksize CHUNKSIZE Chunksize (axis_0) that should be used for the hdf5 output of OrcaSong. [default: 32]
-
- --complib COMPLIB Compression library that should be used for the OrcaSong output.
- All PyTables compression filters are available. [default: zlib]
-
- --complevel COMPLEVEL Compression level that should be used for the OrcaSong output. [default: 1]
-
- --n_bins N_BINS Number of bins that are used in the image making for each dimension, e.g. (x,y,z,t).
- [default: 11,13,18,60]
-
- --det_geo DET_GEO Which detector geometry to use for the binning, e.g. 'Orca_115l_23m_h_9m_v'.
- [default: Orca_115l_23m_h_9m_v]
-
- --do2d If 2D histograms, 'images', should be created.
-
- --do2d_plots If 2D pdf plot visualizations of the 2D histograms should be created, cannot be called if do2d=False.
-
- --do2d_plots_n N For how many events the 2D plot visualizations should be made.
- OrcaSong will exit after reaching N events. [default: 10]
-
- --do3d If 3D histograms, 'images', should be created.
-
- --dont_do4d If 4D histograms, 'images', should NOT be created.
-
- --do4d_mode MODE What dimension should be used in the 4D histograms as the 4th dim.
- Available: 'time', 'channel_id'. [default: time]
+Arguments:
+ FILENAME A KM3NeT hdf5 simfile at JTE level.
- --timecut_mode MODE Defines what timecut mode should be used in hits_to_histograms.py.
- At the moment, these cuts are only optimized for ORCA 115l neutrino events!
- Currently available:
- 'timeslice_relative': Cuts out the central 30% of the snapshot.
- 'trigger_cluster': Cuts based on the mean of the triggered hits.
- The timespan for this cut can be chosen in --timecut_timespan.
- 'None': No timecut.
- [default: trigger_cluster]
+ DETXFILE A .detx geometry file that is associated with the hdf5 file.
- --timecut_timespan TIMESPAN Only used with timecut_mode 'trigger_cluster'.
- Defines the timespan of the trigger_cluster cut.
- Currently available:
- 'all': [-350ns, 850ns] -> 20ns / bin (60 bins)
- 'tight-1': [-250ns, 500ns] -> 12.5ns / bin
- 'tight-2': [-150ns, 200ns] -> 5.8ns / bin
- [default: tight-1]
+ CONFIGFILE A .toml configuration file that contains all configuration options
+ of this script. A default config can be found in the OrcaSong repo:
+ orcasong/default_config.toml
- --do_mc_hits If only the mc_hits (no BG hits!) should be used for the image processing.
-
- --data_cut_triggered If non-triggered hits should be thrown away for the images.
-
- --data_cut_e_low E_LOW Cut events that are lower than the specified threshold value in GeV.
-
- --data_cut_e_high E_HIGH Cut events that are higher than the specified threshold value in GeV.
-
- --data_cut_throw_away FRACTION Throw away a random fraction (percentage) of events. [default: 0.00]
-
- --prod_ident PROD_IDENT Optional int identifier for the used mc production.
- This is useful, if you use events from two different mc productions,
- e.g. the 1-5GeV & 3-100GeV Orca 2016 MC. The prod_ident int will be saved in
- the 'y' dataset of the output file of OrcaSong. [default: 1]
+Options:
+ -h --help Show this screen.
"""
@@ -111,62 +55,71 @@ from orcasong.hits_to_histograms import HistogramMaker
def parse_input():
"""
- Parses and returns all necessary input options for the data_to_images function.
+ Parses and returns all necessary input options for the make_nn_images function.
+
+ Returns
+ -------
+ fname : str
+ Full filepath to the input .h5 file.
+ detx_filepath : str
+ Full filepath to the .detx geometry file that belongs to the fname.
+ config_filepath : str
+ Full filepath to a config file. An example can be found in orcasong/default_config.toml
- Check the data_to_images function to get docs about the individual parameters.
"""
args = docopt(__doc__)
- if args['-c']:
- config = toml.load(args['-c'])
- args.update(config)
-
- fname = args['FILENAME']
+ fname = args['SIMFILE']
detx_filepath = args['DETXFILE']
+ config_filepath = args['CONFIGFILE']
- output_dirpath = args['--o']
- chunksize = int(args['--chunksize'])
- complib = args['--complib']
- complevel = int(args['--complevel'])
- n_bins = tuple(map(int, args['--n_bins'].split(',')))
- det_geo = args['--det_geo']
- do2d = args['--do2d']
- do2d_plots = (args['--do2d_plots'], int(args['--do2d_plots_n']))
- do3d = args['--do3d']
- do4d = (not bool(args['--dont_do4d']), args['--do4d_mode'])
- timecut = (args['--timecut_mode'], args['--timecut_timespan'])
- do_mc_hits = args['--do_mc_hits']
- data_cuts = dict()
- data_cuts['triggered'] = args['--data_cut_triggered']
- data_cuts['energy_lower_limit'] = float(args['--data_cut_e_low']) if args['--data_cut_e_low'] is not None else None
- data_cuts['energy_upper_limit'] = float(args['--data_cut_e_high']) if args['--data_cut_e_high'] is not None else None
- data_cuts['throw_away_prob'] = float(args['--data_cut_throw_away'])
- prod_ident = int(args['--prod_ident'])
+ return fname, detx_filepath, config_filepath
- return fname, detx_filepath, output_dirpath, chunksize, complib, complevel, n_bins, det_geo, do2d,\
- do2d_plots, do3d, do4d, prod_ident, timecut, do_mc_hits, data_cuts
+def load_config(config_filepath):
+ """
+ Loads the config from a .toml file.
+
+ Parameters
+ ----------
+ config_filepath : str
+ Full filepath to a config file. An example can be found in orcasong/default_config.toml
+
+ Returns
+ -------
+ config : dict
+ Dictionary that contains all configuration options of the make_nn_images function.
+ An explanation of the config parameters can be found in orcasong/default_config.toml.
-def parser_check_input(args):
"""
- Sanity check of the user input. Only necessary for options that are not boolean.
+ config = toml.load(config_filepath)
+ print('Loaded the config file from ' + os.path.abspath(config_filepath))
+
+ check_config(config)
+
+ return config
+
+
+def check_config(config):
+ """
+ Sanity check of the user input.
Parameters
----------
- args : dict
- Docopt parser element that contains all input information.
+ config : dict
+ Dictionary that contains all configuration options of the make_nn_images function.
+ An explanation of the config parameters can be found in orcasong/default_config.toml.
"""
#---- Checks input types ----#
# Check for options with a single, non-boolean element
- single_args = {'--det_geo': str, '--do2d_plots_n': int, '--do4d_mode': str, '--timecut_mode': str,
- 'timecut_timespan': str, '--data_cut_e_low ': float, '--data_cut_e_high': float,
- '--data_cut_throw_away': float, '--prod_ident': int}
+ number_args = {'do2d_plots_n': int, 'data_cut_e_low ': float, 'data_cut_e_high': float,
+ 'data_cut_throw_away': float, 'prod_ident': int}
- for key in single_args:
- expected_arg_type = single_args[key]
- parsed_arg = args[key]
+ for key in number_args:
+ expected_arg_type = number_args[key]
+ parsed_arg = config[key]
if parsed_arg is None: # we don't want to check args when there has been no user input
continue
@@ -179,7 +132,7 @@ def parser_check_input(args):
# Checks the n_bins tuple input
try:
- map(int, args['--n_bins'].split(','))
+ map(int, config['n_bins'].split(','))
except ValueError:
raise TypeError('The argument option n_bins only accepts integer values as an input'
' (Format: --n_bins 11,13,18,60).')
@@ -189,16 +142,19 @@ def parser_check_input(args):
# ---- Check other things ----#
- if not os.path.isfile(args['FILENAME']):
- raise IOError('The file -' + args['FILENAME'] + '- does not exist.')
+ if not os.path.isfile(config['SIMFILE']):
+ raise IOError('The file -' + config['SIMFILE'] + '- does not exist.')
+
+ if not os.path.isfile(config['DETXFILE']):
+ raise IOError('The file -' + config['DETXFILE'] + '- does not exist.')
- if not os.path.isfile(args['DETXFILE']):
- raise IOError('The file -' + args['DETXFILE'] + '- does not exist.')
+ if all(do_nd == False for do_nd in [config['do2d'], config['do3d'],config['do4d']]):
+ raise ValueError('At least one of do2d, do3d or do4d options must be set to True.')
- if bool(args['--do2d']) == False and bool(args['--do2d_plots']) == True:
+ if config['do2d'] == False and config['do2d_plots'] == True:
raise ValueError('The 2D pdf images cannot be created if do2d=False!')
- if bool(args['--do2d_plots']) == True and int(args['--do2d_plots_n']) > 100:
+ if config['do2d_plots'] == True and int(config['do2d_plots_n']) > 100:
warnings.warn('You declared do2d_pdf=(True, int) with int > 100. This will take more than two minutes.'
'Do you really want to create pdfs images for so many events?')
@@ -428,8 +384,7 @@ def skip_event(event_track, data_cuts):
return continue_bool
-def data_to_images(fname, detx_filepath, output_dirpath, chunksize, complib, complevel, n_bins, det_geo, do2d, do2d_plots, do3d, do4d, prod_ident, timecut,
- do_mc_hits, data_cuts):
+def make_nn_images(fname, detx_filepath, config):
"""
Main code with config parameters. Reads raw .hdf5 files and creates 2D/3D histogram projections that can be used
for a CNN.
@@ -442,54 +397,36 @@ def data_to_images(fname, detx_filepath, output_dirpath, chunksize, complib, com
String with the full filepath to the corresponding .detx file of the input file.
Used for the binning and for the hits calibration if the input file is not calibrated yet
(e.g. hits do not contain pos_x/y/z, time, ...).
- output_dirpath : str
- Full path to the directory, where the orcasong output should be stored.
- chunksize : int
- Chunksize (along axis_0) that is used for saving the OrcaSong output to a .h5 file.
- complib : str
- Compression library that is used for saving the OrcaSong output to a .h5 file.
- All PyTables compression filters are available, e.g. 'zlib', 'lzf', 'blosc', ... .
- complevel : int
- Compression level for the compression filter that is used for saving the OrcaSong output to a .h5 file.
- n_bins : tuple of int
- Declares the number of bins that should be used for each dimension, e.g. (x,y,z,t).
- det_geo : str
- Declares what detector geometry should be used for the binning. E.g. 'Orca_115l_23m_h_9m_v'.
- do2d : bool
- Declares if 2D histograms, 'images', should be created.
- do2d_plots : tuple(bool, int)
- Declares if pdf visualizations of the 2D histograms should be created, cannot be called if do2d=False.
- The event loop will be stopped after the integer specified in the second argument.
- do3d : bool
- Declares if 3D histograms should be created.
- do4d : tuple(bool, str)
- Tuple that declares if 4D histograms should be created [0] and if yes, what should be used as the 4th dim after xyz.
- Currently, only 'time' and 'channel_id' are available.
- prod_ident : int
- Optional int identifier for the used mc production.
- This is e.g. useful, if you use events from two different mc productions, e.g. the 1-5GeV & 3-100GeV Orca 2016 MC.
- In this case, the events are not fully distinguishable with only the run_id and the event_id!
- In order to keep a separation, an integer can be set in the event_track for all events, such that they stay distinguishable.
- timecut : tuple(str, str/None)
- Tuple that defines what timecut should be used in hits_to_histograms.py.
- Currently available:
- ('timeslice_relative', None): Cuts out the central 30% of the snapshot.
- ('trigger_cluster', 'all' / 'tight-1' / 'tight-2'): Cuts based on the mean of the triggered hits.
- (None, ...): No timecut.
- all: [-350ns, 850ns] -> 20ns / bin (60 bins)
- tight-1: [-250ns, 500ns] -> 12.5ns / bin , tight-2: [-150ns, 200ns] -> 5.8ns / bin
- do_mc_hits : bool
- Declares if hits (False, mc_hits + BG) or mc_hits (True) should be processed.
- data_cuts : dict
- Dictionary that contains information about any possible cuts that should be applied.
- Supports the following cuts: 'triggered', 'energy_lower_limit', 'energy_upper_limit', 'throw_away_prob'.
+ config : dict
+ Dictionary that contains all configuration options of the make_nn_images function.
+ An explanation of the config parameters can be found in orcasong/default_config.toml.
"""
+ # Load all parameters from the config
+ output_dirpath = config['output_dirpath']
+ chunksize, complib, complevel = int(config['chunksize']), int(config['complib']), int(config['complevel'])
+ flush_freq = int(config['flush_freq'])
+ n_bins = tuple(map(int, config['n_bins'].split(',')))
+ timecut, do_mc_hits = config['timecut'], config['do_mc_hits']
+ det_geo = config['det_geo']
+ do2d, do2d_plots = (config['do2d'], int(config['do2d_plots']))
+ do3d = config['do3d']
+ do4d = (config['do4d'], config['do4d_mode'])
+ prod_ident = int(config['prod_ident']) if config['prod_ident'] is not 'None' else None
+ data_cuts = dict()
+ data_cuts['triggered'] = config['data_cut_triggered']
+ data_cuts['energy_lower_limit'] = float(config['data_cut_e_low']) if config['data_cut_e_low'] is not 'None' else None
+ data_cuts['energy_upper_limit'] = float(config['data_cut_e_high']) if config['data_cut_e_high'] is not 'None' else None
+ data_cuts['throw_away_prob'] =float(config['data_cut_throw_away']) if config['data_cut_e_high'] is not 'None' else 0.00
+
make_output_dirs(output_dirpath, do2d, do3d, do4d)
filename = os.path.basename(os.path.splitext(fname)[0])
filename_output = filename.replace('.','_')
- km.GlobalRandomState(seed=42) # set random km3pipe (=numpy) seed
+
+ # set random km3pipe (=numpy) seed
+ km.GlobalRandomState(seed=42)
+ print('Set a Global Random State with the seed < 42 >.')
geo, x_bin_edges, y_bin_edges, z_bin_edges = calculate_bin_edges(n_bins, det_geo, detx_filepath, do4d)
pdf_2d_plots = PdfPages(output_dirpath + '/orcasong_output/4dTo2d/' + filename_output + '_plots.pdf') if do2d_plots[0] is True else None
@@ -500,8 +437,9 @@ def data_to_images(fname, detx_filepath, output_dirpath, chunksize, complib, com
# Initialize OrcaSong Event Pipeline
- pipe = kp.Pipeline()
+ pipe = kp.Pipeline(timeit=True)
pipe.attach(km.common.StatusBar, every=50)
+ pipe.attach(km.common.MemoryObserver, every=50)
pipe.attach(kp.io.hdf5.HDF5Pump, filename=fname)
pipe.attach(km.common.Keep, keys=['EventInfo', 'Header', 'RawHeader', 'McTracks', 'Hits', 'McHits'])
pipe.attach(EventDataExtractor,
@@ -518,18 +456,21 @@ def data_to_images(fname, detx_filepath, output_dirpath, chunksize, complib, com
if do2d:
for proj in ['xy', 'xz', 'yz', 'xt', 'yt', 'zt']:
savestr = output_dirpath + '/orcasong_output/4dTo2d/' + proj + '/' + filename_output + '_' + proj + '.h5'
- pipe.attach(kp.io.HDF5Sink, filename=savestr, blob_keys=[proj, 'event_track'], complib=complib, complevel=complevel, chunksize=chunksize)
+ pipe.attach(kp.io.HDF5Sink, filename=savestr, blob_keys=[proj, 'event_track'], complib=complib,
+ complevel=complevel, chunksize=chunksize, flush_frequency=flush_freq)
if do3d:
for proj in ['xyz', 'xyt', 'xzt', 'yzt', 'rzt']:
savestr = output_dirpath + '/orcasong_output/4dTo3d/' + proj + '/' + filename_output + '_' + proj + '.h5'
- pipe.attach(kp.io.HDF5Sink, filename=savestr, blob_keys=[proj, 'event_track'], complib=complib, complevel=complevel, chunksize=chunksize)
+ pipe.attach(kp.io.HDF5Sink, filename=savestr, blob_keys=[proj, 'event_track'], complib=complib,
+ complevel=complevel, chunksize=chunksize, flush_frequency=flush_freq)
if do4d[0]:
proj = 'xyzt' if not do4d[1] == 'channel_id' else 'xyzc'
savestr = output_dirpath + '/orcasong_output/4dTo4d/' + proj + '/' + filename_output + '_' + proj + '.h5'
- pipe.attach(kp.io.HDF5Sink, filename=savestr, blob_keys=[proj, 'event_track'], complib=complib, complevel=complevel, chunksize=chunksize)
+ pipe.attach(kp.io.HDF5Sink, filename=savestr, blob_keys=[proj, 'event_track'], complib=complib,
+ complevel=complevel, chunksize=chunksize, flush_frequency=flush_freq)
# Execute Pipeline
pipe.drain()
@@ -537,12 +478,11 @@ def data_to_images(fname, detx_filepath, output_dirpath, chunksize, complib, com
def main():
"""
- Parses the input to the main data_to_images function.
+ Parses the input to the main make_nn_images function.
"""
- fname, detx_filepath, output_dirpath, chunksize, complib, complevel, n_bins, det_geo, do2d, do2d_plots, do3d,\
- do4d, prod_ident, timecut, do_mc_hits, data_cuts = parse_input()
- data_to_images(fname, detx_filepath, output_dirpath, chunksize, complib, complevel, n_bins, det_geo, do2d,
- do2d_plots, do3d, do4d, prod_ident, timecut, do_mc_hits, data_cuts)
+ fname, detx_filepath, config_filepath = parse_input()
+ config = load_config(config_filepath)
+ make_nn_images(fname, detx_filepath, config)
if __name__ == '__main__':