Skip to content
Snippets Groups Projects
Commit b81ffd6b authored by ViaFerrata's avatar ViaFerrata
Browse files

Minor.

parent 1f2e0839
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-c_tight-1_60b.toml
+ 8
− 8
View file @ b81ffd6b
......@@ -79,7 +79,7 @@ data_tools_folder = "/home/woody/capn/mppi033h/Code/OrcaSong/orcasong_contrib/da
chunksize = 32
complib = "gzip"
complevel = 1
shuffle_delete = false
shuffle_delete = true
# --- Options, for submitting jobs to concatenate the .list files. --- #
......@@ -87,43 +87,43 @@ shuffle_delete = false
[muon_cc_3_100] # 1 to 2400
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/3-100GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/3-100GeV/xyzc"
run_ids_train = [721, 2400]
run_ids_validate = [1, 720]
[muon_cc_1_5] # 1 to 600
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/1-5GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/1-5GeV/xyzc"
run_ids_train = [181, 600]
run_ids_validate = [1, 180]
[elec_cc_3_100] # 1 to 2400
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/3-100GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/3-100GeV/xyzc"
run_ids_train = [361, 1200]
run_ids_validate = [1, 360]
[elec_cc_1_5] # 1 to 600
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/1-5GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/1-5GeV/xyzc"
run_ids_train = [181, 600]
run_ids_validate = [1, 180]
[elec_nc_3_100] # 1 to 2400
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/3-100GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/3-100GeV/xyzc"
run_ids_train = [361, 1200]
run_ids_validate = [1, 360]
[elec_nc_1_5] # 1 to 600
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/1-5GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/1-5GeV/xyzc"
run_ids_train = [181, 600]
run_ids_validate = [1, 180]
[tau_cc_3_100] # 1 to 1800
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/tau-CC/3-100GeV/xyzc"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/tau-CC/3-100GeV/xyzc"
run_ids_rest = [1, 1800]
# --- Input groups : these are the datafiles, that should be concatenated somehow --- #
\ No newline at end of file
orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-t_tight-1_60b.toml
+ 8
− 8
View file @ b81ffd6b
......@@ -79,7 +79,7 @@ data_tools_folder = "/home/woody/capn/mppi033h/Code/OrcaSong/orcasong_contrib/da
chunksize = 32
complib = "gzip"
complevel = 1
shuffle_delete = false
shuffle_delete = true
# --- Options, for submitting jobs to concatenate the .list files. --- #
......@@ -87,43 +87,43 @@ shuffle_delete = false
[muon_cc_3_100] # 1 to 2400
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/3-100GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/3-100GeV/xyzt"
run_ids_train = [721, 2400]
run_ids_validate = [1, 720]
[muon_cc_1_5] # 1 to 600
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/1-5GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/1-5GeV/xyzt"
run_ids_train = [181, 600]
run_ids_validate = [1, 180]
[elec_cc_3_100] # 1 to 2400
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/3-100GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/3-100GeV/xyzt"
run_ids_train = [361, 1200]
run_ids_validate = [1, 360]
[elec_cc_1_5] # 1 to 600
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/1-5GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/1-5GeV/xyzt"
run_ids_train = [181, 600]
run_ids_validate = [1, 180]
[elec_nc_3_100] # 1 to 2400
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/3-100GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/3-100GeV/xyzt"
run_ids_train = [361, 1200]
run_ids_validate = [1, 360]
[elec_nc_1_5] # 1 to 600
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/1-5GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/1-5GeV/xyzt"
run_ids_train = [181, 600]
run_ids_validate = [1, 180]
[tau_cc_3_100] # 1 to 1800
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/tau-CC/3-100GeV/xyzt"
dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/tau-CC/3-100GeV/xyzt"
run_ids_rest = [1, 1800]
# --- Input groups : these are the datafiles, that should be concatenated somehow --- #
\ No newline at end of file
orcasong_contrib/data_tools/make_data_split/make_data_split.py
+ 6
− 6
View file @ b81ffd6b
......@@ -288,13 +288,13 @@ def make_concatenate_and_shuffle_list_files(cfg):
listfile_fname_wout_ext = os.path.splitext(listfile_fname)[0]
conc_outputfile_fpath = cfg['output_file_folder'] + '/data_split/' + listfile_fname_wout_ext + '.h5'
fpath_bash_script = dirpath + '/job_scripts/submit_concatenate_h5_' + listfile_fname_wout_ext + '.sh'
fpath_bash_script = dirpath + '/job_scripts/concatenate_h5_' + listfile_fname_wout_ext + '.sh'
with open(fpath_bash_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write('#\n')
f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/submit_concatenate_h5_' + listfile_fname_wout_ext + '.out'
' -e ' + cfg['output_file_folder'] + '/logs/submit_concatenate_h5_' + listfile_fname_wout_ext + '.err\n')
f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/concatenate_h5_' + listfile_fname_wout_ext + '.out'
' -e ' + cfg['output_file_folder'] + '/logs/concatenate_h5_' + listfile_fname_wout_ext + '.err\n')
f.write('\n')
f.write('CodeFolder="' + cfg['data_tools_folder'] + '"\n')
f.write('cd ${CodeFolder}\n')
......@@ -321,13 +321,13 @@ def make_concatenate_and_shuffle_list_files(cfg):
# This is the input for the shuffle tool!
conc_outputfile_fpath = cfg['output_file_folder'] + '/data_split/' + listfile_fname_wout_ext + '.h5'
fpath_bash_script = dirpath + '/job_scripts/submit_shuffle_h5_' + listfile_fname_wout_ext + '.sh'
fpath_bash_script = dirpath + '/job_scripts/shuffle_h5_' + listfile_fname_wout_ext + '.sh'
with open(fpath_bash_script, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write('#\n')
f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/submit_shuffle_h5_' + listfile_fname_wout_ext + '.out'
' -e ' + cfg['output_file_folder'] + '/logs/submit_shuffle_h5_' + listfile_fname_wout_ext + '.err\n')
f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/shuffle_h5_' + listfile_fname_wout_ext + '.out'
' -e ' + cfg['output_file_folder'] + '/logs/shuffle_h5_' + listfile_fname_wout_ext + '.err\n')
f.write('\n')
f.write('CodeFolder="' + cfg['data_tools_folder'] + '"\n')
f.write('cd ${CodeFolder}\n')
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment