Skip to content
Snippets Groups Projects
Commit 7ad2cc80 authored by ViaFerrata's avatar ViaFerrata
Browse files

Minor changes.

parent b5630d4a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
docs/conf.py
+ 2
− 2
View file @ 7ad2cc80
...@@ -16,9 +16,9 @@ import os ...@@ -16,9 +16,9 @@ import os
import sys import sys
from datetime import date from datetime import date
from pkg_resources import get_distribution from pkg_resources import get_distribution
sys.path.insert(0, os.path.abspath('../..'))
sys.path.insert(0, os.path.abspath('.'))
import orcasong import orcasong
#sys.path.insert(0, os.path.abspath('.'))
# -- Project information ----------------------------------------------------- # -- Project information -----------------------------------------------------
......
orcasong/make_nn_images.py
+ 5
− 5
View file @ 7ad2cc80
...@@ -314,7 +314,7 @@ def get_file_particle_type(fname): ...@@ -314,7 +314,7 @@ def get_file_particle_type(fname):
String that specifies the type of particles that are contained in the file: ['undefined', 'muon', 'neutrino']. String that specifies the type of particles that are contained in the file: ['undefined', 'muon', 'neutrino'].
""" """
event_pump = kp.io.hdf5.HDF5Pump(filename=fname) # TODO suppress print of hdf5pump and close pump afterwards event_pump = kp.io.hdf5.HDF5Pump(filename=fname, verbose=False) # TODO suppress print of hdf5pump
if 'McTracks' not in event_pump[0]: if 'McTracks' not in event_pump[0]:
file_particle_type = 'undefined' file_particle_type = 'undefined'
...@@ -427,8 +427,8 @@ def make_nn_images(fname, detx_filepath, config): ...@@ -427,8 +427,8 @@ def make_nn_images(fname, detx_filepath, config):
filename_output = filename.replace('.','_') filename_output = filename.replace('.','_')
# set random km3pipe (=numpy) seed # set random km3pipe (=numpy) seed
print('Setting a Global Random State with the seed < 42 >.')
km.GlobalRandomState(seed=42) km.GlobalRandomState(seed=42)
print('Set a Global Random State with the seed < 42 >.')
geo, x_bin_edges, y_bin_edges, z_bin_edges = calculate_bin_edges(n_bins, det_geo, detx_filepath, do4d) geo, x_bin_edges, y_bin_edges, z_bin_edges = calculate_bin_edges(n_bins, det_geo, detx_filepath, do4d)
pdf_2d_plots = PdfPages(output_dirpath + '/orcasong_output/4dTo2d/' + filename_output + '_plots.pdf') if do2d_plots[0] is True else None pdf_2d_plots = PdfPages(output_dirpath + '/orcasong_output/4dTo2d/' + filename_output + '_plots.pdf') if do2d_plots[0] is True else None
...@@ -439,9 +439,9 @@ def make_nn_images(fname, detx_filepath, config): ...@@ -439,9 +439,9 @@ def make_nn_images(fname, detx_filepath, config):
# Initialize OrcaSong Event Pipeline # Initialize OrcaSong Event Pipeline
pipe = kp.Pipeline(timeit=True) pipe = kp.Pipeline() # add timeit=True argument for profiling
pipe.attach(km.common.StatusBar, every=50) pipe.attach(km.common.StatusBar, every=200)
pipe.attach(km.common.MemoryObserver, every=50) pipe.attach(km.common.MemoryObserver, every=400)
pipe.attach(kp.io.hdf5.HDF5Pump, filename=fname) pipe.attach(kp.io.hdf5.HDF5Pump, filename=fname)
pipe.attach(km.common.Keep, keys=['EventInfo', 'Header', 'RawHeader', 'McTracks', 'Hits', 'McHits']) pipe.attach(km.common.Keep, keys=['EventInfo', 'Header', 'RawHeader', 'McTracks', 'Hits', 'McHits'])
pipe.attach(EventDataExtractor, pipe.attach(EventDataExtractor,
......
user/job_submission_scripts/submit_data_to_images.sh
+ 13
− 13
View file @ 7ad2cc80
#!/bin/bash #!/bin/bash
# #
#PBS -l nodes=1:ppn=4:sl,walltime=20:00:00 #PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
#PBS -o /home/woody/capn/mppi033h/logs/orcasong/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.out -e /home/woody/capn/mppi033h/logs/orcasong/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.err #PBS -o /home/woody/capn/mppi033h/logs/orcasong/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.out -e /home/woody/capn/mppi033h/logs/orcasong/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.err
# first non-empty non-comment line ends PBS options # first non-empty non-comment line ends PBS options
# Submit with 'qsub -t 1-x submit_data_to_images.sh' # Submit with 'qsub -t 1-x submit_data_to_images.sh'
# This script uses the data_to_images.py file in order to convert all .h5 raw MC files to .h5 event "images" (CNN input). # This script uses the make_nn_images.py file in order to convert all .h5 raw MC files to .h5 event "images" (CNN input).
# Currently available ORCA 115l sim files: # Currently available ORCA 115l sim files:
# neutrinos: 600 files each for 1-5 GeV prod (muon-CC, elec-CC/NC) number of jobs needed = 5 with files_per_job=120, # neutrinos: 600 files each for 1-5 GeV prod (muon-CC, elec-CC/NC) number of jobs needed = 5 with files_per_job=120,
# else (3-100 GeV prod) # else (3-100 GeV prod)
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# elec-CC = 1200 files, number of jobs needed = 10 with files_per_job=120 # elec-CC = 1200 files, number of jobs needed = 10 with files_per_job=120
# elec-NC = 1200 files, number of jobs needed = 10 with files_per_job=120 # elec-NC = 1200 files, number of jobs needed = 10 with files_per_job=120
# tau-CC = 1800 files (half the n_evts of other interaction channels), number of jobs needed = 15 with files_per_job=120 and half walltime # tau-CC = 1800 files (half the n_evts of other interaction channels), number of jobs needed = 15 with files_per_job=120 and half walltime
# mupage: 20000 files, with files_per_job=250, 80 jobs needed with 20h walltime. # mupage: 20000 files, with files_per_job=200, 100 jobs needed with 5h walltime.
# random_noise: 500 files, with files_per_job=20 , 25 jobs needed with 20h walltime. # random_noise: 500 files, with files_per_job=100 , 5 jobs needed with 5h walltime.
#--- USER INPUT ---# #--- USER INPUT ---#
...@@ -24,10 +24,10 @@ python_env_folder=/home/hpc/capn/mppi033h/.virtualenv/python_3_env/ ...@@ -24,10 +24,10 @@ python_env_folder=/home/hpc/capn/mppi033h/.virtualenv/python_3_env/
job_logs_folder=/home/woody/capn/mppi033h/logs/orcasong/cout job_logs_folder=/home/woody/capn/mppi033h/logs/orcasong/cout
detx_filepath=/home/woody/capn/mppi033h/Code/OrcaSong/user/detx_files/orca_115strings_av23min20mhorizontal_18OMs_alt9mvertical_v1.detx detx_filepath=/home/woody/capn/mppi033h/Code/OrcaSong/user/detx_files/orca_115strings_av23min20mhorizontal_18OMs_alt9mvertical_v1.detx
config_file=/home/woody/capn/mppi033h/Code/OrcaSong/user/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml config_file=/home/woody/capn/mppi033h/Code/OrcaSong/user/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_mupage_xyz-t.toml
particle_type=random_noise particle_type=mupage
mc_prod=random_noise mc_prod=mupage
# total number of files per job # total number of files per job
# For neutrinos 3-100GeV: # For neutrinos 3-100GeV:
...@@ -35,8 +35,8 @@ mc_prod=random_noise ...@@ -35,8 +35,8 @@ mc_prod=random_noise
# For neutrinos 1-5GeV: # For neutrinos 1-5GeV:
# muon-CC/elec-CC/elec-NC n=120 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00 # muon-CC/elec-CC/elec-NC n=120 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
# For mupage: n=250 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00 # For mupage: n=250 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
# For random_noise: n=20 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00 # For random_noise: n=100 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
files_per_job=20 # must be dividible by 4! files_per_job=250 # must be dividible by 4!
#--- USER INPUT ---# #--- USER INPUT ---#
...@@ -87,9 +87,9 @@ do ...@@ -87,9 +87,9 @@ do
thread3=$((${file_no_loop_start} + 2)) thread3=$((${file_no_loop_start} + 2))
thread4=$((${file_no_loop_start} + 3)) thread4=$((${file_no_loop_start} + 3))
(time taskset -c 0 make_nn_images -c ${config_file} ${folder}/${filename}.${thread1}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread1}.txt) & (time taskset -c 0 make_nn_images ${folder}/${filename}.${thread1}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread1}.txt) &
(time taskset -c 1 make_nn_images -c ${config_file} ${folder}/${filename}.${thread2}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread2}.txt) & (time taskset -c 1 make_nn_images ${folder}/${filename}.${thread2}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread2}.txt) &
(time taskset -c 2 make_nn_images -c ${config_file} ${folder}/${filename}.${thread3}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread3}.txt) & (time taskset -c 2 make_nn_images ${folder}/${filename}.${thread3}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread3}.txt) &
(time taskset -c 3 make_nn_images -c ${config_file} ${folder}/${filename}.${thread4}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread4}.txt) & (time taskset -c 3 make_nn_images ${folder}/${filename}.${thread4}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread4}.txt) &
wait wait
done done
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment