- Improved usability of job submission script

- Add folder structure to config files folder

Jenkinsfile

-#!groovy
-CHAT_CHANNEL = '#deep_learning'
-DEVELOPERS = ['mmoser@km3net.de, michael.m.moser@fau.de']
-
-def projectProperties = [
-        disableConcurrentBuilds(),
-        gitLabConnection('KM3NeT GitLab')
-]
-properties(projectProperties)
-
-node('master') {
-
-    // Start with a clean workspace
-    // cleanWs()
-    checkout scm
-
-    def docker_image = docker.build("orcasong-container:${env.BUILD_ID}")
-
-    docker_image.inside("-u root:root") {
-        gitlabBuilds(builds: ["install", "test"]) {
-            updateGitlabCommitStatus name: 'install', state: 'pending'
-            stage('build') {
-                try {
-                    sh """
-                        pip install .
-                    """
-                    updateGitlabCommitStatus name: 'install', state: 'success'
-                } catch (e) {
-                    // sendChatMessage("Build Failed")
-                    sendMail("Build Failed")
-                    updateGitlabCommitStatus name: 'install', state: 'failed'
-                    throw e
-                }
-            }
-            updateGitlabCommitStatus name: 'test', state: 'pending'
-            stage('test') {
-                try {
-                    sh """
-                        python -c "import orcasong"
-                    """
-                    updateGitlabCommitStatus name: 'test', state: 'success'
-                } catch (e) {
-                    // sendChatMessage("Build Failed")
-                    sendMail("Build Failed")
-                    updateGitlabCommitStatus name: 'test', state: 'failed'
-                    throw e
-                }
-            }
-        }
-    }
-}
-
-
-def sendChatMessage(message, channel=CHAT_CHANNEL) {
-    rocketSend channel: channel, message: "${message} - [Build ${env.BUILD_NUMBER} ](${env.BUILD_URL})"
-}
-
-
-def sendMail(subject, message='', developers=DEVELOPERS) {
-    for (int i = 0; i < developers.size(); i++) {
-        def developer = DEVELOPERS[i]
-        emailext (
-            subject: "$subject - Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
-            body: """
-                <p>$message</p>
-                <p>Check console output at <a href ='${env.BUILD_URL}'>${env.BUILD_URL}</a> to view the results.</p>
-            """,
-            to: developer
-        )
-    }
-}
-

orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_1-5GeV_xyz-c.toml

+# A config file for OrcaSong with multiple configurations.
+# Outcomment the config that you want to use!
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+### All available options with some dummy values
+# --n_bins = '11,13,18,60'
+# --det_geo = 'Orca_115l_23m_h_9m_v'
+# --do2d = false
+# --do2d_plots = false
+# --do2d_plots_n = 10
+# --do3d = false
+# --do4d = true
+# --do4d_mode = 'time'
+# --timecut_mode = 'trigger_cluster'
+# --timecut_timespan = 'tight_1'
+# --do_mc_hits = false
+# --data_cut_triggered = false
+# --data_cut_e_low = 3
+# --data_cut_e_high = 100
+# --data_cut_throw_away = 0.00
+# --prod_ident = 1
+
+
+####----- Configuration for ORCA 115l -----####
+
+###--- 1-5GeV ---###
+
+### only take < 3GeV events, info throw away: elec-CC 0.25, muon-CC 0.25, elec-NC: 0.00
+
+## XYZ-C
+
+--n_bins = '11,13,18,31'
+--det_geo = 'Orca_115l_23m_h_9m_v'
+--do4d_mode = 'channel_id'
+--timecut_mode = 'trigger_cluster'
+--timecut_timespan = 'tight_0'
+--prod_ident = 2 # only for neutrinos: 1: 3-100 GeV prod, 2: 1-5 GeV prod.
\ No newline at end of file

orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_1-5GeV_xyz-t.toml

+# A config file for OrcaSong with multiple configurations.
+# Outcomment the config that you want to use!
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+### All available options with some dummy values
+# --n_bins = '11,13,18,60'
+# --det_geo = 'Orca_115l_23m_h_9m_v'
+# --do2d = false
+# --do2d_plots = false
+# --do2d_plots_n = 10
+# --do3d = false
+# --do4d = true
+# --do4d_mode = 'time'
+# --timecut_mode = 'trigger_cluster'
+# --timecut_timespan = 'tight_1'
+# --do_mc_hits = false
+# --data_cut_triggered = false
+# --data_cut_e_low = 3
+# --data_cut_e_high = 100
+# --data_cut_throw_away = 0.00
+# --prod_ident = 1
+
+
+####----- Configuration for ORCA 115l -----####
+
+###--- 1-5GeV ---###
+
+### only take < 3GeV events, info throw away: elec-CC 0.25, muon-CC 0.25, elec-NC: 0.00
+
+## XYZ-T
+
+--n_bins = '11,13,18,100'
+--det_geo = 'Orca_115l_23m_h_9m_v'
+--do4d_mode = 'time'
+--timecut_mode = 'trigger_cluster'
+--timecut_timespan = 'tight_0'
+--prod_ident = 2 # only for neutrinos: 1: 3-100 GeV prod, 2: 1-5 GeV prod.
\ No newline at end of file

orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_3-100GeV_xyz-c.toml

+# A config file for OrcaSong with multiple configurations.
+# Outcomment the config that you want to use!
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+### All available options with some dummy values
+# --n_bins = '11,13,18,60'
+# --det_geo = 'Orca_115l_23m_h_9m_v'
+# --do2d = false
+# --do2d_plots = false
+# --do2d_plots_n = 10
+# --do3d = false
+# --do4d = true
+# --do4d_mode = 'time'
+# --timecut_mode = 'trigger_cluster'
+# --timecut_timespan = 'tight_1'
+# --do_mc_hits = false
+# --data_cut_triggered = false
+# --data_cut_e_low = 3
+# --data_cut_e_high = 100
+# --data_cut_throw_away = 0.00
+# --prod_ident = 1
+
+
+####----- Configuration for ORCA 115l -----####
+
+###--- 3-100GeV ---###
+
+## XYZ-C
+
+--n_bins = '11,13,18,31'
+--det_geo = 'Orca_115l_23m_h_9m_v'
+--do4d_mode = 'channel_id'
+--timecut_mode = 'trigger_cluster'
+--timecut_timespan = 'tight_0'
+--prod_ident = 1 # only for neutrinos: 1: 3-100 GeV prod, 2: 1-5 GeV prod.
\ No newline at end of file

orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_3-100GeV_xyz-t.toml

+# A config file for OrcaSong with multiple configurations.
+# Outcomment the config that you want to use!
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+### All available options with some dummy values
+# --n_bins = '11,13,18,60'
+# --det_geo = 'Orca_115l_23m_h_9m_v'
+# --do2d = false
+# --do2d_plots = false
+# --do2d_plots_n = 10
+# --do3d = false
+# --do4d = true
+# --do4d_mode = 'time'
+# --timecut_mode = 'trigger_cluster'
+# --timecut_timespan = 'tight_1'
+# --do_mc_hits = false
+# --data_cut_triggered = false
+# --data_cut_e_low = 3
+# --data_cut_e_high = 100
+# --data_cut_throw_away = 0.00
+# --prod_ident = 1
+
+
+####----- Configuration for ORCA 115l -----####
+
+###--- 3-100GeV ---###
+
+## XYZ-T
+
+--n_bins = '11,13,18,100'
+--det_geo = 'Orca_115l_23m_h_9m_v'
+--do4d_mode = 'time'
+--timecut_mode = 'trigger_cluster'
+--timecut_timespan = 'tight_0'
+--prod_ident = 1 # only for neutrinos: 1: 3-100 GeV prod, 2: 1-5 GeV prod.
\ No newline at end of file

orcasong/submit_h5_data_to_h5_input.sh → orcasong/job_submission_scripts/submit_data_to_images.sh

@@ -9,17 +9,45 @@
 # The total amount of simulated files for each event type in ORCA is 600 -> file 1-600
 # The files should be converted in batches of files_per_job=60 files per job
 
-# load env
-source activate /home/hpc/capn/mppi033h/.virtualenv/python_3_env/
+#--- USER INPUT ---##
+
+# load env, only working for conda env as of now
+python_env_folder=/home/hpc/capn/mppi033h/.virtualenv/python_3_env/
+code_folder=/home/woody/capn/mppi033h/Code/OrcaSong/orcasong
+
+detx_filepath=${code_folder}/detx_files/orca_115strings_av23min20mhorizontal_18OMs_alt9mvertical_v1.detx
+config_file=${code_folder}/config/orca_115l_regression/conf_ORCA_115l_3-100GeV_xyz-t.toml
+
+particle_type=muon-CC
+mc_prod=neutr_3-100GeV
+
+files_per_job=60 # total number of files per job, e.g. 10 jobs for 600: 600/10 = 60
+
+#--- USER INPUT ---##
 
 n=${PBS_ARRAYID}
+source activate ${python_env_folder}
+cd ${code_folder}
 
-CodeFolder=/home/woody/capn/mppi033h/Code/OrcaSong
-cd ${CodeFolder}
+declare -A filename_arr
+declare -A folder_ip_files_arr
+
+filename_arr=( ["muon-CC"]="JTE.KM3Sim.gseagen.muon-CC.3-100GeV-9.1E7-1bin-3.0gspec.ORCA115_9m_2016"
+               ["elec-CC"]="JTE.KM3Sim.gseagen.elec-CC.3-100GeV-1.1E6-1bin-3.0gspec.ORCA115_9m_2016"
+               ["elec-NC"]="JTE.KM3Sim.gseagen.elec-NC.3-100GeV-3.4E6-1bin-3.0gspec.ORCA115_9m_2016"
+               ["tau-CC"]="JTE.KM3Sim.gseagen.tau-CC.3.4-100GeV-2.0E8-1bin-3.0gspec.ORCA115_9m_2016"
+               ["mupage"]="JTE.ph.ph.mupage.ph.ph.ph.ORCA115_9m_2016")
+
+folder_ip_files_arr=( ["neutr_3-100GeV"]="/home/saturn/capn/mppi033h/Data/raw_data/ORCA_JTE_NEMOWATER/calibrated/with_jte_times/3-100GeV/${particle_type}"
+                      ["neutr_1-5GeV"]="/home/saturn/capn/mppi033h/Data/raw_data/ORCA_JTE_NEMOWATER/calibrated/with_jte_times/1-5GeV/${particle_type}"
+                      ["mupage"]="/home/saturn/capn/mppi033h/Data/raw_data/mupage")
+
+filename="${filename_arr[${particle_type}]}"
+folder="${folder_ip_files_arr[${mc_prod}]}"
 
 #ParticleType=muon-CC
 #ParticleType=elec-CC
-ParticleType=elec-NC
+#ParticleType=elec-NC
 #ParticleType=tau-CC
 # ----- 3-100GeV------
 #FileName=JTE.KM3Sim.gseagen.muon-CC.3-100GeV-9.1E7-1bin-3.0gspec.ORCA115_9m_2016 #muon-CC
@@ -31,26 +59,10 @@ ParticleType=elec-NC
 # ----- 1-5GeV------
 #FileName=JTE.KM3Sim.gseagen.muon-CC.1-5GeV-9.2E5-1bin-1.0gspec.ORCA115_9m_2016 #muon-CC
 #FileName=JTE.KM3Sim.gseagen.elec-CC.1-5GeV-2.7E5-1bin-1.0gspec.ORCA115_9m_2016 #elec-CC
-FileName=JTE.KM3Sim.gseagen.elec-NC.1-5GeV-2.2E6-1bin-1.0gspec.ORCA115_9m_2016 #elec-NC
-HDFFOLDER=/home/woody/capn/mppi033h/Data/ORCA_JTE_NEMOWATER/raw_data/calibrated/with_jte_times/1-5GeV/${ParticleType}
+#FileName=JTE.KM3Sim.gseagen.elec-NC.1-5GeV-2.2E6-1bin-1.0gspec.ORCA115_9m_2016 #elec-NC
+#HDFFOLDER=/home/woody/capn/mppi033h/Data/ORCA_JTE_NEMOWATER/raw_data/calibrated/with_jte_times/1-5GeV/${ParticleType}
 # ----- 1-5GeV------
 
-# -- denser detector study --
-# elec-CC 170 files, muon-CC 400 files.
-# 15m: 400 m-CC files, 170 e-CC files. time/file: m-CC 30s, e-CC 1m --> files_per_job_m-CC = 80, files_per_job_e-CC = 36, 5 jobs
-# 12m: 400 m-CC files, 170 e-CC files. time/file: m-CC 30s, e-CC 1m10s --> files_per_job_m-CC = 80, files_per_job_e-CC = 36, 5 jobs
-# 9m: 400 m-CC files, 170 e-CC files. time/file: m-CC 40s, e-CC - --> files_per_job_m-CC = 80, files_per_job_e-CC = 36, 5 jobs
-# 6m: 400 m-CC files, 170 e-CC files. time/file: m-CC 53s, e-CC - --> files_per_job_m-CC = 80, files_per_job_e-CC = 36, 5 jobs
-# 4.5m: 400 m-CC files, 170 e-CC files. time/file: m-CC 1m3s, e-CC - --> files_per_job_m-CC = 80, files_per_job_e-CC = 36, 5 jobs
-#vert_space=4p5m # 15m fpj m: 80, e: 36 ; 12m fpj m: , e: ;
-#FileName=JTE.KM3Sim.selectedEventsInCylinder_gseagen.muon-CC.1-20GeV-2.0E8-1bin-3.0gspec.ORCA115_h23v${vert_space}_2016 #muon-CC
-#FileName=JTE.KM3Sim.selectedEventsInCylinder_gseagen.elec-CC.1-20GeV-1.0E6-1bin-3.0gspec.ORCA115_h23v${vert_space}_2016 #elec-CC
-#HDFFOLDER=/home/woody/capn/mppi033h/Data/ORCA_JTE_NEMOWATER/raw_data/calibrated/without_mc_time_fix/denser_detector_study/${vert_space}
-#HDFFOLDER=/home/vault/capn/mppi033h/4p5m # 4p5m
-# -- denser detector study --
-
-files_per_job=60 # total number of files per job, e.g. 10 jobs for 600: 600/10 = 60
-
 # run
 no_of_loops=$((${files_per_job}/4)) # divide by 4 cores -> e.g, 15 4-core loops needed for files_per_job=60
 file_no_start=$((1+((${n}-1) * ${files_per_job}))) # filenumber of the first file that is being processed by this script (depends on JobArray variable 'n')
@@ -63,10 +75,10 @@ do
     thread3=$((${file_no_loop_start} + 2))
     thread4=$((${file_no_loop_start} + 3))
 
-    (time taskset -c 0  python ${CodeFolder}/h5_data_to_h5_input.py ${HDFFOLDER}/${FileName}.${thread1}.h5 > ./logs/cout/${FileName}.${thread1}.txt) &
-    (time taskset -c 1  python ${CodeFolder}/h5_data_to_h5_input.py ${HDFFOLDER}/${FileName}.${thread2}.h5 > ./logs/cout/${FileName}.${thread2}.txt) &
-    (time taskset -c 2  python ${CodeFolder}/h5_data_to_h5_input.py ${HDFFOLDER}/${FileName}.${thread3}.h5 > ./logs/cout/${FileName}.${thread3}.txt) &
-    (time taskset -c 3  python ${CodeFolder}/h5_data_to_h5_input.py ${HDFFOLDER}/${FileName}.${thread4}.h5 > ./logs/cout/${FileName}.${thread4}.txt) &
+    (time taskset -c 0  python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread1}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread1}.txt) &
+    (time taskset -c 1  python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread2}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread2}.txt) &
+    (time taskset -c 2  python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread3}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread3}.txt) &
+    (time taskset -c 3  python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread4}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread4}.txt) &
     wait
 done