From 28e1da54d1b2c790670c8e0baf540c78bcaafb9a Mon Sep 17 00:00:00 2001
From: Daniel Guderian <dguderian@km3net.de>
Date: Tue, 26 Jan 2021 21:00:16 +0100
Subject: [PATCH] change to fixed 1GB ram use in shuffle scripts
---
orcasong/tools/make_data_split.py | 2 +-
tests/test_make_data_split.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/orcasong/tools/make_data_split.py b/orcasong/tools/make_data_split.py
index 1510277..b7646e1 100644
--- a/orcasong/tools/make_data_split.py
+++ b/orcasong/tools/make_data_split.py
@@ -357,7 +357,7 @@ def make_concatenate_and_shuffle_scripts(cfg):
f.write('\n')
f.write('# Shuffle the h5 file \n')
- f.write('h5shuffle2 ' + conc_outputfile_fpath + ' --max_ram 2000000000 \n') #fix to 2GB ram; in lyon using a fraction
+ f.write('h5shuffle2 ' + conc_outputfile_fpath + ' --max_ram 1000000000 \n') #fix to 1GB ram; in lyon using a fraction
#is difficult...
#time python shuffle/shuffle_h5.py'
#+ delete_flag_shuffle_tool
diff --git a/tests/test_make_data_split.py b/tests/test_make_data_split.py
index 228d26c..c6f0934 100644
--- a/tests/test_make_data_split.py
+++ b/tests/test_make_data_split.py
@@ -41,7 +41,7 @@ class TestMakeDataSplit(TestCase):
cls.file_path_list_val = ['processed_data_neutrino/processed_graph_neutrino.h5','processed_data_neutrino/processed_graph_neutrino.h5\n']
cls.n_events_list = [18,3]
cls.contents_concatenate_script = ['concatenate ' + list_output_train + ' --outfile ' + concatenate_file]
- cls.contents_shuffle_script = ['h5shuffle2 ' + concatenate_file + ' --max_ram 2000000000 \n']
+ cls.contents_shuffle_script = ['h5shuffle2 ' + concatenate_file + ' --max_ram 1000000000 \n']
#create list_file_dir
--
GitLab