Merge remote-tracking branch 'origin/master' into concatenate_scripts

orcasong/tools/concatenate.py

+import os
 import time
 import h5py
 import numpy as np
@@ -177,6 +178,7 @@ class FileConcatenator:
 
         errors, rows_per_file, valid_input_files = [], [0], []
         for i, file_name in enumerate(input_files, start=1):
+            file_name = os.path.abspath(file_name)
             try:
                 rows_this_file = _get_rows(file_name, keys_stripped)
             except Exception as e:

setup.py

@@ -29,7 +29,7 @@ setup(
     entry_points={'console_scripts': [
         'concatenate=orcasong.tools.concatenate:main',
         'h5shuffle=orcasong.tools.postproc:h5shuffle',
-        'h5shuffle2=orcasong.tools.shuffle2:h5shuffle2',
+        'h5shuffle2=orcasong.tools.shuffle2:run_parser',
         'plot_binstats=orcasong.plotting.plot_binstats:main',
         'make_nn_images=legacy.make_nn_images:main',
         'make_dsplit=orcasong_contrib.data_tools.make_data_split.make_data_split:main']}

tests/test_postproc.py

@@ -3,7 +3,7 @@ import os
 import h5py
 import numpy as np
 import orcasong.tools.postproc as postproc
-from orcasong.tools.shuffle2 import shuffle_v2
+import orcasong.tools.shuffle2 as shuffle2
 
 __author__ = 'Stefan Reck'
 
@@ -41,11 +41,10 @@ class TestShuffleV2(TestCase):
 
         self.x, self.y = _make_shuffle_dummy_file(self.temp_input)
         np.random.seed(42)
-        shuffle_v2(
+        shuffle2.h5shuffle2(
             input_file=self.temp_input,
             output_file=self.temp_output,
             datasets=("x", "y"),
-            chunks=True,
             max_ram=400,  # -> 2 batches
         )
 
@@ -76,6 +75,20 @@ class TestShuffleV2(TestCase):
                 f["x"][:, 0], target_order
             )
 
+    def test_run_3_iterations(self):
+        # just check if it goes through without errors
+        fname = "temp_output_triple.h5"
+        try:
+            shuffle2.h5shuffle2(
+                input_file=self.temp_input,
+                output_file=fname,
+                datasets=("x", "y"),
+                iterations=3,
+            )
+        finally:
+            if os.path.exists(fname):
+                os.remove(fname)
+
 
 def _make_shuffle_dummy_file(filepath):
     x = np.random.rand(22, 2)