From 7ad2cc807ad9887ee2a8543ef490314cf18c800b Mon Sep 17 00:00:00 2001
From: ViaFerrata <michimoser@onlinehome.de>
Date: Mon, 14 Jan 2019 12:14:46 +0100
Subject: [PATCH] Minor changes.
---
docs/conf.py | 4 +--
orcasong/make_nn_images.py | 10 +++----
.../submit_data_to_images.sh | 26 +++++++++----------
3 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/docs/conf.py b/docs/conf.py
index 49c973d..b61b2a3 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -16,9 +16,9 @@ import os
import sys
from datetime import date
from pkg_resources import get_distribution
-sys.path.insert(0, os.path.abspath('../..'))
-sys.path.insert(0, os.path.abspath('.'))
+
import orcasong
+#sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
diff --git a/orcasong/make_nn_images.py b/orcasong/make_nn_images.py
index d581e30..5d7112d 100644
--- a/orcasong/make_nn_images.py
+++ b/orcasong/make_nn_images.py
@@ -314,7 +314,7 @@ def get_file_particle_type(fname):
String that specifies the type of particles that are contained in the file: ['undefined', 'muon', 'neutrino'].
"""
- event_pump = kp.io.hdf5.HDF5Pump(filename=fname) # TODO suppress print of hdf5pump and close pump afterwards
+ event_pump = kp.io.hdf5.HDF5Pump(filename=fname, verbose=False) # TODO suppress print of hdf5pump
if 'McTracks' not in event_pump[0]:
file_particle_type = 'undefined'
@@ -427,8 +427,8 @@ def make_nn_images(fname, detx_filepath, config):
filename_output = filename.replace('.','_')
# set random km3pipe (=numpy) seed
+ print('Setting a Global Random State with the seed < 42 >.')
km.GlobalRandomState(seed=42)
- print('Set a Global Random State with the seed < 42 >.')
geo, x_bin_edges, y_bin_edges, z_bin_edges = calculate_bin_edges(n_bins, det_geo, detx_filepath, do4d)
pdf_2d_plots = PdfPages(output_dirpath + '/orcasong_output/4dTo2d/' + filename_output + '_plots.pdf') if do2d_plots[0] is True else None
@@ -439,9 +439,9 @@ def make_nn_images(fname, detx_filepath, config):
# Initialize OrcaSong Event Pipeline
- pipe = kp.Pipeline(timeit=True)
- pipe.attach(km.common.StatusBar, every=50)
- pipe.attach(km.common.MemoryObserver, every=50)
+ pipe = kp.Pipeline() # add timeit=True argument for profiling
+ pipe.attach(km.common.StatusBar, every=200)
+ pipe.attach(km.common.MemoryObserver, every=400)
pipe.attach(kp.io.hdf5.HDF5Pump, filename=fname)
pipe.attach(km.common.Keep, keys=['EventInfo', 'Header', 'RawHeader', 'McTracks', 'Hits', 'McHits'])
pipe.attach(EventDataExtractor,
diff --git a/user/job_submission_scripts/submit_data_to_images.sh b/user/job_submission_scripts/submit_data_to_images.sh
index 76789bd..97d4355 100644
--- a/user/job_submission_scripts/submit_data_to_images.sh
+++ b/user/job_submission_scripts/submit_data_to_images.sh
@@ -1,11 +1,11 @@
#!/bin/bash
#
-#PBS -l nodes=1:ppn=4:sl,walltime=20:00:00
+#PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
#PBS -o /home/woody/capn/mppi033h/logs/orcasong/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.out -e /home/woody/capn/mppi033h/logs/orcasong/submit_data_to_images_${PBS_JOBID}_${PBS_ARRAYID}.err
# first non-empty non-comment line ends PBS options
# Submit with 'qsub -t 1-x submit_data_to_images.sh'
-# This script uses the data_to_images.py file in order to convert all .h5 raw MC files to .h5 event "images" (CNN input).
+# This script uses the make_nn_images.py file in order to convert all .h5 raw MC files to .h5 event "images" (CNN input).
# Currently available ORCA 115l sim files:
# neutrinos: 600 files each for 1-5 GeV prod (muon-CC, elec-CC/NC) number of jobs needed = 5 with files_per_job=120,
# else (3-100 GeV prod)
@@ -13,8 +13,8 @@
# elec-CC = 1200 files, number of jobs needed = 10 with files_per_job=120
# elec-NC = 1200 files, number of jobs needed = 10 with files_per_job=120
# tau-CC = 1800 files (half the n_evts of other interaction channels), number of jobs needed = 15 with files_per_job=120 and half walltime
-# mupage: 20000 files, with files_per_job=250, 80 jobs needed with 20h walltime.
-# random_noise: 500 files, with files_per_job=20 , 25 jobs needed with 20h walltime.
+# mupage: 20000 files, with files_per_job=200, 100 jobs needed with 5h walltime.
+# random_noise: 500 files, with files_per_job=100 , 5 jobs needed with 5h walltime.
#--- USER INPUT ---#
@@ -24,10 +24,10 @@ python_env_folder=/home/hpc/capn/mppi033h/.virtualenv/python_3_env/
job_logs_folder=/home/woody/capn/mppi033h/logs/orcasong/cout
detx_filepath=/home/woody/capn/mppi033h/Code/OrcaSong/user/detx_files/orca_115strings_av23min20mhorizontal_18OMs_alt9mvertical_v1.detx
-config_file=/home/woody/capn/mppi033h/Code/OrcaSong/user/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_random_noise_xyz-c.toml
+config_file=/home/woody/capn/mppi033h/Code/OrcaSong/user/config/orca_115l_mupage_rn_neutr_classifier/conf_ORCA_115l_mupage_xyz-t.toml
-particle_type=random_noise
-mc_prod=random_noise
+particle_type=mupage
+mc_prod=mupage
# total number of files per job
# For neutrinos 3-100GeV:
@@ -35,8 +35,8 @@ mc_prod=random_noise
# For neutrinos 1-5GeV:
# muon-CC/elec-CC/elec-NC n=120 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
# For mupage: n=250 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
-# For random_noise: n=20 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
-files_per_job=20 # must be dividible by 4!
+# For random_noise: n=100 with PBS -l nodes=1:ppn=4:sl,walltime=5:00:00
+files_per_job=250 # must be dividible by 4!
#--- USER INPUT ---#
@@ -87,9 +87,9 @@ do
thread3=$((${file_no_loop_start} + 2))
thread4=$((${file_no_loop_start} + 3))
- (time taskset -c 0 make_nn_images -c ${config_file} ${folder}/${filename}.${thread1}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread1}.txt) &
- (time taskset -c 1 make_nn_images -c ${config_file} ${folder}/${filename}.${thread2}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread2}.txt) &
- (time taskset -c 2 make_nn_images -c ${config_file} ${folder}/${filename}.${thread3}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread3}.txt) &
- (time taskset -c 3 make_nn_images -c ${config_file} ${folder}/${filename}.${thread4}.h5 ${detx_filepath} > ${job_logs_folder}/${filename}.${thread4}.txt) &
+ (time taskset -c 0 make_nn_images ${folder}/${filename}.${thread1}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread1}.txt) &
+ (time taskset -c 1 make_nn_images ${folder}/${filename}.${thread2}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread2}.txt) &
+ (time taskset -c 2 make_nn_images ${folder}/${filename}.${thread3}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread3}.txt) &
+ (time taskset -c 3 make_nn_images ${folder}/${filename}.${thread4}.h5 ${detx_filepath} ${config_file} > ${job_logs_folder}/${filename}.${thread4}.txt) &
wait
done
\ No newline at end of file
--
GitLab