inference issue on ARCA6 file
Hi @sreck, @nvandermeulen has ran the inference on several ARCA6 muon files, but with this specific file:
/sps/km3net/users/nvanderm/ML_project/muon_files/ML_mcv6.3.mupage_100G.sirene.jterbr00009636.jchain.aashower.7.h5
it gives the following error:
2022-06-13 15:35:16.482478: W tensorflow/core/framework/op_kernel.cc:1733] INVALID_ARGUMENT: required broadcastable shapes
Traceback (most recent call last):
File "/usr/local/bin/orcanet", line 8, in <module>
sys.exit(main())
File "/usr/local/lib/python3.8/dist-packages/orcanet/parser.py", line 251, in main
func(**kwargs)
File "/usr/local/lib/python3.8/dist-packages/orcanet/parser.py", line 85, in inference
return orga.inference(epoch=epoch, fileno=fileno)
File "/usr/local/lib/python3.8/dist-packages/orcanet/core.py", line 399, in inference
return [filename for filename in gen]
File "/usr/local/lib/python3.8/dist-packages/orcanet/core.py", line 399, in <listcomp>
return [filename for filename in gen]
File "/usr/local/lib/python3.8/dist-packages/orcanet/core.py", line 428, in _inference
backend.h5_inference(
File "/usr/local/lib/python3.8/dist-packages/orcanet/backend.py", line 237, in h5_inference
y_pred = model.predict_on_batch(info_blob["xs"])
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1986, in predict_on_batch
outputs = self.predict_function(iterator)
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py", line 153, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py", line 58, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.InvalidArgumentError: 4 root error(s) found.
(0) INVALID_ARGUMENT: required broadcastable shapes
[[node replica_2/model/batch_normalization/batchnorm/mul_1
(defined at /usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py:910)
]]
[[replica_2/model/ts_output/Softmax/_831]]
(1) INVALID_ARGUMENT: required broadcastable shapes
[[node replica_2/model/batch_normalization/batchnorm/mul_1
(defined at /usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py:910)
]]
[[model/tf.RaggedTensor.from_row_splits_3/RaggedFromRowSplits/RowPartitionFromRowSplits/assert_non_negative/assert_less_equal/Assert/Assert/_243]]
(2) INVALID_ARGUMENT: required broadcastable shapes
[[node replica_2/model/batch_normalization/batchnorm/mul_1
(defined at /usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py:910)
]]
[[model/tf.RaggedTensor.from_row_splits_1/RaggedFromRowSplits/control_dependency/_152]]
(3) INVALID_ARGUMENT: required broadcastable shapes
[[node replica_2/model/batch_normalization/batchnorm/mul_1
(defined at /usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py:910)
]]
0 successful operations.
0 derived errors ignored. [Op:__inference_predict_function_9170]
Errors may have originated from an input operation.
Input Source operations connected to node replica_2/model/batch_normalization/batchnorm/mul_1:
In[0] RaggedFromVariant_5/RaggedTensorFromVariant (defined at /usr/local/lib/python3.8/dist-packages/keras/engine/training.py:1610)
In[1] replica_2/model/batch_normalization/batchnorm/mul:
Operation defined at: (most recent call last)
>>> File "/usr/lib/python3.8/threading.py", line 890, in _bootstrap
>>> self._bootstrap_inner()
>>>
>>> File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
>>> self.run()
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1604, in run_step
>>> outputs = model.predict_step(data)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1572, in predict_step
>>> return self(x, training=False)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 451, in call
>>> return self._run_internal_graph(
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 589, in _run_internal_graph
>>> outputs = node.layer(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py", line 910, in call
>>> outputs = tf.nn.batch_normalization(inputs, _broadcast(mean),
>>>
Input Source operations connected to node replica_2/model/batch_normalization/batchnorm/mul_1:
In[0] RaggedFromVariant_5/RaggedTensorFromVariant (defined at /usr/local/lib/python3.8/dist-packages/keras/engine/training.py:1610)
In[1] replica_2/model/batch_normalization/batchnorm/mul:
Operation defined at: (most recent call last)
>>> File "/usr/lib/python3.8/threading.py", line 890, in _bootstrap
>>> self._bootstrap_inner()
>>>
>>> File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
>>> self.run()
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1604, in run_step
>>> outputs = model.predict_step(data)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1572, in predict_step
>>> return self(x, training=False)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 451, in call
>>> return self._run_internal_graph(
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 589, in _run_internal_graph
>>> outputs = node.layer(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py", line 910, in call
>>> outputs = tf.nn.batch_normalization(inputs, _broadcast(mean),
>>>
Input Source operations connected to node replica_2/model/batch_normalization/batchnorm/mul_1:
In[0] RaggedFromVariant_5/RaggedTensorFromVariant (defined at /usr/local/lib/python3.8/dist-packages/keras/engine/training.py:1610)
In[1] replica_2/model/batch_normalization/batchnorm/mul:
Operation defined at: (most recent call last)
>>> File "/usr/lib/python3.8/threading.py", line 890, in _bootstrap
>>> self._bootstrap_inner()
>>>
>>> File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
>>> self.run()
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1604, in run_step
>>> outputs = model.predict_step(data)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1572, in predict_step
>>> return self(x, training=False)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 451, in call
>>> return self._run_internal_graph(
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 589, in _run_internal_graph
>>> outputs = node.layer(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py", line 910, in call
>>> outputs = tf.nn.batch_normalization(inputs, _broadcast(mean),
>>>
Input Source operations connected to node replica_2/model/batch_normalization/batchnorm/mul_1:
In[0] RaggedFromVariant_5/RaggedTensorFromVariant (defined at /usr/local/lib/python3.8/dist-packages/keras/engine/training.py:1610)
In[1] replica_2/model/batch_normalization/batchnorm/mul:
Operation defined at: (most recent call last)
>>> File "/usr/lib/python3.8/threading.py", line 890, in _bootstrap
>>> self._bootstrap_inner()
>>>
>>> File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
>>> self.run()
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1604, in run_step
>>> outputs = model.predict_step(data)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1572, in predict_step
>>> return self(x, training=False)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 451, in call
>>> return self._run_internal_graph(
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/functional.py", line 589, in _run_internal_graph
>>> outputs = node.layer(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py", line 1083, in __call__
>>> outputs = call_fn(inputs, *args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
>>> return fn(*args, **kwargs)
>>>
>>> File "/usr/local/lib/python3.8/dist-packages/keras/layers/normalization/batch_normalization.py", line 910, in call
>>> outputs = tf.nn.batch_normalization(inputs, _broadcast(mean),
>>>
Function call stack:
predict_function -> predict_function -> predict_function -> predict_function
Exception ignored in: <function Pool.__del__ at 0x2b26d0a33160>
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/pool.py", line 268, in __del__
File "/usr/lib/python3.8/multiprocessing/queues.py", line 362, in put
AttributeError: 'NoneType' object has no attribute 'dumps'
we do not understand what is the issue with that file as it seems to be ok. Have you seen this error before?