diff --git a/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-c_tight-1_60b.toml b/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-c_tight-1_60b.toml index c541b794ba15927fcfcb11f40d6dd53a0bd8ef5f..2562c0ae159e46ac0f373a4e5e3355f7d314ae1f 100644 --- a/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-c_tight-1_60b.toml +++ b/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-c_tight-1_60b.toml @@ -79,7 +79,7 @@ data_tools_folder = "/home/woody/capn/mppi033h/Code/OrcaSong/orcasong_contrib/da chunksize = 32 complib = "gzip" complevel = 1 -shuffle_delete = false +shuffle_delete = true # --- Options, for submitting jobs to concatenate the .list files. --- # @@ -87,43 +87,43 @@ shuffle_delete = false [muon_cc_3_100] # 1 to 2400 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/3-100GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/3-100GeV/xyzc" run_ids_train = [721, 2400] run_ids_validate = [1, 720] [muon_cc_1_5] # 1 to 600 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/1-5GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/1-5GeV/xyzc" run_ids_train = [181, 600] run_ids_validate = [1, 180] [elec_cc_3_100] # 1 to 2400 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/3-100GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/3-100GeV/xyzc" run_ids_train = [361, 1200] run_ids_validate = [1, 360] [elec_cc_1_5] # 1 to 600 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/1-5GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/1-5GeV/xyzc" run_ids_train = [181, 600] run_ids_validate = [1, 180] [elec_nc_3_100] # 1 to 2400 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/3-100GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/3-100GeV/xyzc" run_ids_train = [361, 1200] run_ids_validate = [1, 360] [elec_nc_1_5] # 1 to 600 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/1-5GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/1-5GeV/xyzc" run_ids_train = [181, 600] run_ids_validate = [1, 180] [tau_cc_3_100] # 1 to 1800 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/tau-CC/3-100GeV/xyzc" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/tau-CC/3-100GeV/xyzc" run_ids_rest = [1, 1800] # --- Input groups : these are the datafiles, that should be concatenated somehow --- # \ No newline at end of file diff --git a/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-t_tight-1_60b.toml b/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-t_tight-1_60b.toml index 21ff3e4d7825893f39ca160e9dfe560a06987199..ffb4aa2f5a30821aa72aa81e4697c26d67fb6dd4 100644 --- a/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-t_tight-1_60b.toml +++ b/orcasong_contrib/data_tools/make_data_split/configs/config_regression_xyz-t_tight-1_60b.toml @@ -79,7 +79,7 @@ data_tools_folder = "/home/woody/capn/mppi033h/Code/OrcaSong/orcasong_contrib/da chunksize = 32 complib = "gzip" complevel = 1 -shuffle_delete = false +shuffle_delete = true # --- Options, for submitting jobs to concatenate the .list files. --- # @@ -87,43 +87,43 @@ shuffle_delete = false [muon_cc_3_100] # 1 to 2400 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/3-100GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/3-100GeV/xyzt" run_ids_train = [721, 2400] run_ids_validate = [1, 720] [muon_cc_1_5] # 1 to 600 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/muon-CC/1-5GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/muon-CC/1-5GeV/xyzt" run_ids_train = [181, 600] run_ids_validate = [1, 180] [elec_cc_3_100] # 1 to 2400 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/3-100GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/3-100GeV/xyzt" run_ids_train = [361, 1200] run_ids_validate = [1, 360] [elec_cc_1_5] # 1 to 600 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-CC/1-5GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-CC/1-5GeV/xyzt" run_ids_train = [181, 600] run_ids_validate = [1, 180] [elec_nc_3_100] # 1 to 2400 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/3-100GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/3-100GeV/xyzt" run_ids_train = [361, 1200] run_ids_validate = [1, 360] [elec_nc_1_5] # 1 to 600 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/elec-NC/1-5GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/elec-NC/1-5GeV/xyzt" run_ids_train = [181, 600] run_ids_validate = [1, 180] [tau_cc_3_100] # 1 to 1800 -dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_0_100b_t_bg_classifier/tau-CC/3-100GeV/xyzt" +dir = "/home/saturn/capn/mppi033h/Data/input_images/ORCA_2016_115l/tight_1_60b_regression_classifier/tau-CC/3-100GeV/xyzt" run_ids_rest = [1, 1800] # --- Input groups : these are the datafiles, that should be concatenated somehow --- # \ No newline at end of file diff --git a/orcasong_contrib/data_tools/make_data_split/make_data_split.py b/orcasong_contrib/data_tools/make_data_split/make_data_split.py index 8bfe7637c7ae85c68c38c5d4ba800a599e3a62b6..b066f8d9cb9d2635bd2217d6b99a50919ff5597a 100644 --- a/orcasong_contrib/data_tools/make_data_split/make_data_split.py +++ b/orcasong_contrib/data_tools/make_data_split/make_data_split.py @@ -288,13 +288,13 @@ def make_concatenate_and_shuffle_list_files(cfg): listfile_fname_wout_ext = os.path.splitext(listfile_fname)[0] conc_outputfile_fpath = cfg['output_file_folder'] + '/data_split/' + listfile_fname_wout_ext + '.h5' - fpath_bash_script = dirpath + '/job_scripts/submit_concatenate_h5_' + listfile_fname_wout_ext + '.sh' + fpath_bash_script = dirpath + '/job_scripts/concatenate_h5_' + listfile_fname_wout_ext + '.sh' with open(fpath_bash_script, 'w') as f: f.write('#!/usr/bin/env bash\n') f.write('#\n') - f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/submit_concatenate_h5_' + listfile_fname_wout_ext + '.out' - ' -e ' + cfg['output_file_folder'] + '/logs/submit_concatenate_h5_' + listfile_fname_wout_ext + '.err\n') + f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/concatenate_h5_' + listfile_fname_wout_ext + '.out' + ' -e ' + cfg['output_file_folder'] + '/logs/concatenate_h5_' + listfile_fname_wout_ext + '.err\n') f.write('\n') f.write('CodeFolder="' + cfg['data_tools_folder'] + '"\n') f.write('cd ${CodeFolder}\n') @@ -321,13 +321,13 @@ def make_concatenate_and_shuffle_list_files(cfg): # This is the input for the shuffle tool! conc_outputfile_fpath = cfg['output_file_folder'] + '/data_split/' + listfile_fname_wout_ext + '.h5' - fpath_bash_script = dirpath + '/job_scripts/submit_shuffle_h5_' + listfile_fname_wout_ext + '.sh' + fpath_bash_script = dirpath + '/job_scripts/shuffle_h5_' + listfile_fname_wout_ext + '.sh' with open(fpath_bash_script, 'w') as f: f.write('#!/usr/bin/env bash\n') f.write('#\n') - f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/submit_shuffle_h5_' + listfile_fname_wout_ext + '.out' - ' -e ' + cfg['output_file_folder'] + '/logs/submit_shuffle_h5_' + listfile_fname_wout_ext + '.err\n') + f.write('#PBS -o ' + cfg['output_file_folder'] + '/logs/shuffle_h5_' + listfile_fname_wout_ext + '.out' + ' -e ' + cfg['output_file_folder'] + '/logs/shuffle_h5_' + listfile_fname_wout_ext + '.err\n') f.write('\n') f.write('CodeFolder="' + cfg['data_tools_folder'] + '"\n') f.write('cd ${CodeFolder}\n')