diff --git a/orcasong/tools/make_data_split.py b/orcasong/tools/make_data_split.py index 15102776360127420a13798dddee9cadb41a25ea..b7646e1ed735bef5fb39ac8b6b792b72ae27edfd 100644 --- a/orcasong/tools/make_data_split.py +++ b/orcasong/tools/make_data_split.py @@ -357,7 +357,7 @@ def make_concatenate_and_shuffle_scripts(cfg): f.write('\n') f.write('# Shuffle the h5 file \n') - f.write('h5shuffle2 ' + conc_outputfile_fpath + ' --max_ram 2000000000 \n') #fix to 2GB ram; in lyon using a fraction + f.write('h5shuffle2 ' + conc_outputfile_fpath + ' --max_ram 1000000000 \n') #fix to 1GB ram; in lyon using a fraction #is difficult... #time python shuffle/shuffle_h5.py' #+ delete_flag_shuffle_tool diff --git a/tests/test_make_data_split.py b/tests/test_make_data_split.py index 228d26c8284d25e2c6b4072975e57208b0666396..c6f09344d1840999c1f6beacbd6d492d7eb4b874 100644 --- a/tests/test_make_data_split.py +++ b/tests/test_make_data_split.py @@ -41,7 +41,7 @@ class TestMakeDataSplit(TestCase): cls.file_path_list_val = ['processed_data_neutrino/processed_graph_neutrino.h5','processed_data_neutrino/processed_graph_neutrino.h5\n'] cls.n_events_list = [18,3] cls.contents_concatenate_script = ['concatenate ' + list_output_train + ' --outfile ' + concatenate_file] - cls.contents_shuffle_script = ['h5shuffle2 ' + concatenate_file + ' --max_ram 2000000000 \n'] + cls.contents_shuffle_script = ['h5shuffle2 ' + concatenate_file + ' --max_ram 1000000000 \n'] #create list_file_dir