diff --git a/orcasong/tools/concatenate.py b/orcasong/tools/concatenate.py index 74f9d74ffe14eecff4d2098783527bc3f30d62ce..4f3ed156bd67edf900856d3c6beffd9e3d919000 100644 --- a/orcasong/tools/concatenate.py +++ b/orcasong/tools/concatenate.py @@ -1,3 +1,4 @@ +import os import time import h5py import numpy as np @@ -177,6 +178,7 @@ class FileConcatenator: errors, rows_per_file, valid_input_files = [], [0], [] for i, file_name in enumerate(input_files, start=1): + file_name = os.path.abspath(file_name) try: rows_this_file = _get_rows(file_name, keys_stripped) except Exception as e: diff --git a/setup.py b/setup.py index f956c809c3cc4e90b82bf7e52611e3edce18d90a..100ea4c680088b19837952c19fe7722c0d91447e 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( entry_points={'console_scripts': [ 'concatenate=orcasong.tools.concatenate:main', 'h5shuffle=orcasong.tools.postproc:h5shuffle', - 'h5shuffle2=orcasong.tools.shuffle2:h5shuffle2', + 'h5shuffle2=orcasong.tools.shuffle2:run_parser', 'plot_binstats=orcasong.plotting.plot_binstats:main', 'make_nn_images=legacy.make_nn_images:main', 'make_dsplit=orcasong_contrib.data_tools.make_data_split.make_data_split:main']} diff --git a/tests/test_postproc.py b/tests/test_postproc.py index abb9e8d55f14a2d12fe571e1bde8cfa8002191e2..85a461e3fe2b5c007e60ac0aa0c830a1fcdb98dd 100644 --- a/tests/test_postproc.py +++ b/tests/test_postproc.py @@ -3,7 +3,7 @@ import os import h5py import numpy as np import orcasong.tools.postproc as postproc -from orcasong.tools.shuffle2 import shuffle_v2 +import orcasong.tools.shuffle2 as shuffle2 __author__ = 'Stefan Reck' @@ -41,11 +41,10 @@ class TestShuffleV2(TestCase): self.x, self.y = _make_shuffle_dummy_file(self.temp_input) np.random.seed(42) - shuffle_v2( + shuffle2.h5shuffle2( input_file=self.temp_input, output_file=self.temp_output, datasets=("x", "y"), - chunks=True, max_ram=400, # -> 2 batches ) @@ -76,6 +75,20 @@ class TestShuffleV2(TestCase): f["x"][:, 0], target_order ) + def test_run_3_iterations(self): + # just check if it goes through without errors + fname = "temp_output_triple.h5" + try: + shuffle2.h5shuffle2( + input_file=self.temp_input, + output_file=fname, + datasets=("x", "y"), + iterations=3, + ) + finally: + if os.path.exists(fname): + os.remove(fname) + def _make_shuffle_dummy_file(filepath): x = np.random.rand(22, 2)