From d8afb52076e29c760e735524f693db9ab89934d7 Mon Sep 17 00:00:00 2001
From: ViaFerrata <michimoser@onlinehome.de>
Date: Fri, 7 Dec 2018 15:49:49 +0100
Subject: [PATCH] Add random_noise config.
---
.../conf_ORCA_115l_random_noise_xyz-c.toml | 35 +++++++++++++++
.../conf_ORCA_115l_random_noise_xyz-t.toml | 35 +++++++++++++++
.../submit_data_to_images.sh | 43 +++++++++++++------
3 files changed, 101 insertions(+), 12 deletions(-)
create mode 100644 orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml
create mode 100644 orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-t.toml
diff --git a/orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml b/orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml
new file mode 100644
index 0000000..ab87ea9
--- /dev/null
+++ b/orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml
@@ -0,0 +1,35 @@
+# A config file for OrcaSong with multiple configurations.
+# Outcomment the config that you want to use!
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+### All available options with some dummy values
+# --n_bins = '11,13,18,60'
+# --det_geo = 'Orca_115l_23m_h_9m_v'
+# --do2d = false
+# --do2d_plots = false
+# --do2d_plots_n = 10
+# --do3d = false
+# --do4d = true
+# --do4d_mode = 'time'
+# --timecut_mode = 'trigger_cluster'
+# --timecut_timespan = 'tight_1'
+# --do_mc_hits = false
+# --data_cut_triggered = false
+# --data_cut_e_low = 3
+# --data_cut_e_high = 100
+# --data_cut_throw_away = 0.00
+# --prod_ident = 1
+
+
+####----- Configuration for ORCA 115l -----####
+
+###--- 3-100GeV ---###
+
+## XYZ-C
+
+--n_bins = '11,13,18,31'
+--det_geo = 'Orca_115l_23m_h_9m_v'
+--do4d_mode = 'channel_id'
+--timecut_mode = 'trigger_cluster'
+--timecut_timespan = 'tight_0'
+--prod_ident = 4 # for neutrinos: 1: 3-100 GeV prod, 2: 1-5 GeV prod ; mupage: 3 ; random_noise: 4
\ No newline at end of file
diff --git a/orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-t.toml b/orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-t.toml
new file mode 100644
index 0000000..1db8329
--- /dev/null
+++ b/orcasong/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-t.toml
@@ -0,0 +1,35 @@
+# A config file for OrcaSong with multiple configurations.
+# Outcomment the config that you want to use!
+# More info about the .toml format at https://github.com/toml-lang/toml
+
+### All available options with some dummy values
+# --n_bins = '11,13,18,60'
+# --det_geo = 'Orca_115l_23m_h_9m_v'
+# --do2d = false
+# --do2d_plots = false
+# --do2d_plots_n = 10
+# --do3d = false
+# --do4d = true
+# --do4d_mode = 'time'
+# --timecut_mode = 'trigger_cluster'
+# --timecut_timespan = 'tight_1'
+# --do_mc_hits = false
+# --data_cut_triggered = false
+# --data_cut_e_low = 3
+# --data_cut_e_high = 100
+# --data_cut_throw_away = 0.00
+# --prod_ident = 1
+
+
+####----- Configuration for ORCA 115l -----####
+
+###--- 3-100GeV ---###
+
+## XYZ-T
+
+--n_bins = '11,13,18,100'
+--det_geo = 'Orca_115l_23m_h_9m_v'
+--do4d_mode = 'time'
+--timecut_mode = 'trigger_cluster'
+--timecut_timespan = 'tight_0'
+--prod_ident = 4 # for neutrinos: 1: 3-100 GeV prod, 2: 1-5 GeV prod ; mupage: 3 ; random_noise: 4
\ No newline at end of file
diff --git a/orcasong/job_submission_scripts/submit_data_to_images.sh b/orcasong/job_submission_scripts/submit_data_to_images.sh
index 6f42f3c..a25de70 100644
--- a/orcasong/job_submission_scripts/submit_data_to_images.sh
+++ b/orcasong/job_submission_scripts/submit_data_to_images.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-#PBS -l nodes=1:ppn=4:sl32g,walltime=1:30:00
+#PBS -l nodes=1:ppn=4:sl32g,walltime=06:00:00
#PBS -o /home/woody/capn/mppi033h/logs/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.out -e /home/woody/capn/mppi033h/logs/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.err
# first non-empty non-comment line ends PBS options
@@ -12,7 +12,8 @@
# elec-CC = 1200 files, number of jobs needed = 20,
# elec-NC = 1200 files, number of jobs needed = 20,
# tau-CC = 1800 files (half the size of other interaction channels), number of jobs needed = 30 (half walltime)
-# mupage: 20000 files, TODO
+# mupage: 20000 files, with files_per_job=1000 20 jobs needed.
+# random_noise: 500 files, with files_per_job=
#--- USER INPUT ---##
@@ -22,12 +23,17 @@ python_env_folder=/home/hpc/capn/mppi033h/.virtualenv/python_3_env/
code_folder=/home/woody/capn/mppi033h/Code/OrcaSong/orcasong
detx_filepath=${code_folder}/detx_files/orca_115strings_av23min20mhorizontal_18OMs_alt9mvertical_v1.detx
-config_file=${code_folder}/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_1-5GeV_xyz-t.toml
+config_file=${code_folder}/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml
-particle_type=elec-NC
-mc_prod=neutr_1-5GeV
+particle_type=random_noise
+mc_prod=random_noise
-files_per_job=60 # total number of files per job, e.g. 10 jobs for 600: 600/10 = 60
+# total number of files per job, e.g. 10 jobs for 600: 600/10 = 60. For neutrin
+# For neutrinos and random_noise n=60 with PBS -l nodes=1:ppn=4:sl32g,walltime=3:00:00
+# For mupage: n=1000 with PBS -l nodes=1:ppn=4:sl32g,walltime=15:00:00
+files_per_job=50
+
+n_cores=2 # number of available CPU cores, be careful about memory as well! Currently only working for 4 cores!
#--- USER INPUT ---##
@@ -66,21 +72,34 @@ folder="${folder_ip_files_arr[${mc_prod}]}"
# run
-no_of_loops=$((${files_per_job}/4)) # divide by 4 cores -> e.g, 15 4-core loops needed for files_per_job=60
+no_of_loops=$((${files_per_job}/${n_cores})) # divide by 4 cores -> e.g, 15 4-core loops needed for files_per_job=60
file_no_start=$((1+((${n}-1) * ${files_per_job}))) # filenumber of the first file that is being processed by this script (depends on JobArray variable 'n')
+# currently only working for 4 cores
+
+#for (( k=1; k<=${no_of_loops}; k++ ))
+#do
+# file_no_loop_start=$((${file_no_start}+(k-1)*${n_cores}))
+# thread1=${file_no_loop_start}
+# thread2=$((${file_no_loop_start} + 1))
+# thread3=$((${file_no_loop_start} + 2))
+# thread4=$((${file_no_loop_start} + 3))
+#
+# (time taskset -c 0 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread1}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread1}.txt) &
+# (time taskset -c 1 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread2}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread2}.txt) &
+# (time taskset -c 2 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread3}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread3}.txt) &
+# (time taskset -c 3 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread4}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread4}.txt) &
+# wait
+#done
+
for (( k=1; k<=${no_of_loops}; k++ ))
do
- file_no_loop_start=$((${file_no_start}+(k-1)*4))
+ file_no_loop_start=$((${file_no_start}+(k-1)*${n_cores}))
thread1=${file_no_loop_start}
thread2=$((${file_no_loop_start} + 1))
- thread3=$((${file_no_loop_start} + 2))
- thread4=$((${file_no_loop_start} + 3))
(time taskset -c 0 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread1}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread1}.txt) &
(time taskset -c 1 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread2}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread2}.txt) &
- (time taskset -c 2 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread3}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread3}.txt) &
- (time taskset -c 3 python ${code_folder}/data_to_images.py -c ${config_file} ${folder}/${filename}.${thread4}.h5 ${detx_filepath} > ./job_logs/cout/${filename}.${thread4}.txt) &
wait
done
--
GitLab