add fix

orcasong/tools/concatenate.py

@@ -148,6 +148,12 @@ class FileConcatenator:
                     # if chunk is an integer, its the first dimension and same
                     #  chunks for all datasets
                     chunks = (chunks,) + folder_data.shape[1:]
+                if (isinstance(chunks, int) and chunks > dset_shape[0]) or chunks[0] > dset_shape[0]:
+                    warnings.warn(
+                        f"Specified chunkshape {chunks} is larger than dataset! "
+                        f"Setting chunksize to {dset_shape} instead."
+                    )
+                    chunks = dset_shape
                 f_out.create_dataset(
                     dset_name,
                     shape=dset_shape,

tests/test_concatenate.py

@@ -115,7 +115,9 @@ class TestFileConcatenator(unittest.TestCase):
                 np.testing.assert_array_equal(target, f["rec_array"][()])
 
     def test_chunksize_larger_than_data(self):
-        fc = conc.FileConcatenator(self.dummy_files, comptopts_update={"chunksize": 100})
+        fc = conc.FileConcatenator(self.dummy_files, comptopts_update={
+            "chunksize": {"numpy_array": (100, 7, 3), "rec_array": 200}
+        })
         with tempfile.TemporaryFile() as tf:
             fc.concatenate(tf)
             with h5py.File(tf, "r") as f: