From b781d9a1db394a6383bb150d246f577c1889249c Mon Sep 17 00:00:00 2001
From: Stefan Reck <stefan.reck@fau.de>
Date: Thu, 28 Jan 2021 11:43:54 +0100
Subject: [PATCH] minor changes and move stuff around
---
Makefile | 1 -
docs/getting_started.rst | 10 ++++------
docs/tools.rst | 17 +++++++++++++----
.../make_data_split_config.toml | 0
orcasong_contrib/__init__.py | 0
setup.py | 4 ++--
6 files changed, 19 insertions(+), 13 deletions(-)
rename orcasong/tools/make_data_split_configs/example_make_data_split_config.toml => examples/make_data_split_config.toml (100%)
delete mode 100644 orcasong_contrib/__init__.py
diff --git a/Makefile b/Makefile
index b914a01..792309d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,5 @@
PKGNAME=orcasong
ALLNAMES = $(PKGNAME)
-ALLNAMES += orcasong_contrib
install:
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index 7811b3b..fd0babb 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -28,12 +28,6 @@ instructions on how to do this.
The resulting DL files can already be used as input for networks!
-Step 2.2: Quickly define which files to concatenate
----------------------------------------------------
-If wanted, a list with all DL files that should go into one specific file
-can be produced with :ref:`make_data_split`. Here, the directories and run_ids
-making up the train and validation sets can be set in a config.
-
Step 3: Concatenate
-------------------
Mandatory for training files, recommended for everything else.
@@ -47,6 +41,10 @@ See :ref:`concatenate` for details.
X runs for your training set. Instead, choose runs randomly over
the whole period.
+.. note::
+ For mixing e.g. neutrinos and muon, a list with all DL files that should
+ go into one specific file
+ can be produced with :ref:`make_data_split`.
Step 4: Shuffle
---------------
diff --git a/docs/tools.rst b/docs/tools.rst
index e49e170..022f980 100644
--- a/docs/tools.rst
+++ b/docs/tools.rst
@@ -8,9 +8,18 @@ Orcasong comes with some tools to further process data.
Make_data_split
---------------
-Create datasets for different tasks (like classification or regression) from the files resulting from OrcaSong, based on the run_id. This is particularly helpful for a run-by-run data analysis or to generate equally large datasets per class. A toml config is used, in which the directories and ranges of runs to be considered can be specified, as well as the subdivision into training and validation sets. Detailed descriptions for the options available can be found in the example config in the subfolder make_data_split_configs. As output, a list in txt format with the filepaths belonging to one set is created that can be passed to the concatenate for creating one single file out of the many.
-
-In fact, with the option make_qsub_bash_files, scripts for the concatenation and shuffle, to be directly submitted on computing clusters, are created.
+Create datasets for different tasks (like classification or regression) from the files
+resulting from OrcaSong, based on the run_id. This is particularly helpful
+for a run-by-run data analysis or to generate equally large datasets per class.
+A toml config is used, in which the directories and ranges of runs to be considered
+can be specified, as well as the subdivision into training and validation sets.
+Detailed descriptions for the options available can be found in examples/make_data_split_config.toml.
+As output, a list in txt format with
+the filepaths belonging to one set is created that can be passed to the concatenate
+for creating one single file out of the many.
+
+In fact, with the option make_qsub_bash_files, scripts for the concatenation
+and shuffle, to be directly submitted on computing clusters, are created.
Can be used via the commandline::
@@ -60,4 +69,4 @@ or import function for general postprocessing:
postproc_file(output_filepath_concat)
-
+Theres also a faster (beta) version available called h5shuffle2.
diff --git a/orcasong/tools/make_data_split_configs/example_make_data_split_config.toml b/examples/make_data_split_config.toml
similarity index 100%
rename from orcasong/tools/make_data_split_configs/example_make_data_split_config.toml
rename to examples/make_data_split_config.toml
diff --git a/orcasong_contrib/__init__.py b/orcasong_contrib/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/setup.py b/setup.py
index 100ea4c..e7c3320 100644
--- a/setup.py
+++ b/setup.py
@@ -30,9 +30,9 @@ setup(
'concatenate=orcasong.tools.concatenate:main',
'h5shuffle=orcasong.tools.postproc:h5shuffle',
'h5shuffle2=orcasong.tools.shuffle2:run_parser',
+ 'make_dsplit=orcasong.tools.make_data_split:main',
'plot_binstats=orcasong.plotting.plot_binstats:main',
- 'make_nn_images=legacy.make_nn_images:main',
- 'make_dsplit=orcasong_contrib.data_tools.make_data_split.make_data_split:main']}
+ ]}
)
__author__ = 'Stefan Reck, Michael Moser, Daniel Guderian'
--
GitLab