Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • km3py/km3io
  • spenamartinez/km3io
2 results
Show changes
Commits on Source (773)
[run]
source = km3io
[report]
omit =
src/km3io/definitions/*
src/km3io/version.py
exclude_lines =
pragma: no cover
raise AssertionError
raise NotImplementedError
if 0:
if __name__ == .__main__.:
if self.debug:
if settings.DEBUG
def __repr__
@njit
@nb.njit
......@@ -6,50 +6,21 @@ __pycache__/
# C extensions
*.so
# Version info for PyPI
km3noroot/version.txt
# Distribution / packaging
.Python
env/
bin/
venv/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
.eggs
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
.coverage.*
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Rope
.ropeproject
reports
# Sphinx documentation
doc/_build/
......@@ -57,28 +28,12 @@ doc/auto_examples/
doc/modules/
doc/api
# VI swap files
*.swp
*.swo
# PyCharm files
.idea
# jupyter files
.ipynb_checkpoints/
#
junit*.xml
reports
# misc
*~
*.dqd
*.dat
.pytest_cache/
.DS_Store
junit.xml
.mypy_cache
# venv, pyenv tmp
.python-version
venv
# Version info
src/km3io/version.py
......@@ -4,49 +4,67 @@ image: docker.km3net.de/base/python:3
# only cache local items.
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
CONTAINER_TEST_IMAGE: docker.km3net.de/test/km3io:$CI_COMMIT_REF_SLUG
CONTAINER_RELEASE_IMAGE: docker.km3net.de/km3io:$CI_COMMIT_TAG
CONTAINER_LATEST_IMAGE: docker.km3net.de/km3io:latest
SINGULARITY_RELEASE_IMAGE: km3io_${CI_COMMIT_TAG}.sif
cache:
paths:
- .cache/pip
- venv/
key: "$CI_COMMIT_REF_SLUG"
stages:
- test
- coverage
- doc
- docker
- singularity
- release
.virtualenv_template: &virtualenv_definition |
python -V
pip install virtualenv
virtualenv venv
python3 -m venv venv
source venv/bin/activate
pip install -r requirements-dev.txt
make install
pip install -U pip setuptools wheel setuptools_scm
hash -r
pip freeze
make install-dev
pip freeze
.junit_template: &junit_definition
artifacts:
reports:
junit: "reports/junit*.xml"
test-py3.5:
image: docker.km3net.de/base/python:3.5
test-py3.9:
image: docker.km3net.de/base/python:3.9
stage: test
script:
- *virtualenv_definition
- make test
<<: *junit_definition
test-py3.6:
image: docker.km3net.de/base/python:3.6
test-py3.10:
image: docker.km3net.de/base/python:3.10
stage: test
script:
- *virtualenv_definition
- make test
<<: *junit_definition
test-py3.7:
image: docker.km3net.de/base/python:3.7
test-py3.11:
image: docker.km3net.de/base/python:3.11
stage: test
script:
- *virtualenv_definition
- make test
<<: *junit_definition
test-py3.12:
image: git.km3net.de:4567/common/dockerfiles/base/python:3.12
stage: test
script:
- *virtualenv_definition
......@@ -54,18 +72,19 @@ test-py3.7:
<<: *junit_definition
code-style:
image: docker.km3net.de/base/python:3.7
image: git.km3net.de:4567/common/dockerfiles/base/python:3.12
stage: test
script:
- *virtualenv_definition
- yapf -r -d -e "venv" .
- make black-check
allow_failure: true
coverage:
image: docker.km3net.de/base/python:3.6
image: git.km3net.de:4567/common/dockerfiles/base/python:3.12
stage: coverage
script:
- *virtualenv_definition
- make install-dev
- "make test-cov|grep TOTAL| awk '{printf \"COVERAGE: %.2f%%\", (1-$3/$2)*100 }'"
coverage: '/COVERAGE:\s*([0-9]*\.[0-9]*%)/'
# - make test-cov
......@@ -74,8 +93,17 @@ coverage:
paths:
- reports/coverage
build-docs:
image: git.km3net.de:4567/common/dockerfiles/base/python:3.12
stage: doc
script:
- *virtualenv_definition
- cd doc && make clean && make html
cache: {}
pages:
image: docker.km3net.de/base/python:3.6
image: git.km3net.de:4567/common/dockerfiles/base/python:3.12
stage: doc
script:
- *virtualenv_definition
......@@ -89,3 +117,70 @@ pages:
only:
- tags
- master
docker:
image: docker:stable
services:
- docker:dind
stage: docker
script:
- docker build --pull -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE
tags:
- docker
only:
- tags
pypi:
image: git.km3net.de:4567/common/dockerfiles/base/python:3.12
stage: release
cache: {}
script:
- pip install -U twine wheel
- python setup.py sdist
- python setup.py bdist_wheel
- twine upload dist/*
only:
- tags
cc-lyon:
image: docker.km3net.de/base/ci-helper:1
stage: release
script:
- mkdir -p /root/.ssh && chmod 700 /root/.ssh
- ssh-keyscan -H $IN2P3_HOST > ~/.ssh/known_hosts
- chmod 600 /root/.ssh/known_hosts
- sshpass -p $IN2P3_PASSWORD ssh -v $IN2P3_USERNAME@$IN2P3_HOST 'export KM3NET_THRONG_DIR=/pbs/throng/km3net; export MODULEPATH=/pbs/throng/km3net/modulefiles; . /usr/share/Modules/init/bash; module load km3net_env; module load python/3.9.20; pip install git+https://git.km3net.de/km3py/km3io.git'
only:
- tags
- master
allow_failure: true
release-docker:
image: docker:stable
services:
- docker:dind
stage: release
script:
- docker pull $CONTAINER_TEST_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_LATEST_IMAGE
- docker push $CONTAINER_RELEASE_IMAGE
- docker push $CONTAINER_LATEST_IMAGE
tags:
- docker
only:
- tags
release-singularity:
image: docker.km3net.de/base/singularity:3.6.2
stage: release
script:
- singularity build $SINGULARITY_RELEASE_IMAGE docker://$CONTAINER_TEST_IMAGE
- du -sh $SINGULARITY_RELEASE_IMAGE
- mkdir -p ~/.ssh && ssh-keyscan -H -t rsa $KM3NET_SFTP_HOST >> ~/.ssh/known_hosts
- apt install -q lftp
- lftp -u $KM3NET_SFTP_USER,$KM3NET_SFTP_PASSWORD sftp://$KM3NET_SFTP_HOST -e "set ssl:verify-certificate no; cd singularity/; put $SINGULARITY_RELEASE_IMAGE; bye"
only:
- tags
Unreleased changes
------------------
Version 1
---------
1.2.4 / 2024-03-12
~~~~~~~~~~~~~~~~~~
* Version bump for conda forge
1.2.0 / 2024-06-24
~~~~~~~~~~~~~~~~~~
* Removed online format support (online events, timeslices and summary slices) in favour of
the `KM3io.jl <https://git.km3net.de/common/KM3io.jl>`__` Julia Package.
* uproot 5 and awkward 2 are now required
* Python 3.7+ required
1.1.0 / 2024-03-14
~~~~~~~~~~~~~~~~~~
* A few astro helpers were added: azimuth(), zenith(), phi(), theta(), ...
1.0.2 / 2023-10-08
~~~~~~~~~~~~~~~~~~
* Deprecation fixes
1.0.1 / 2023-06-15
~~~~~~~~~~~~~~~~~~
* Minor changes
1.0.0 / 2023-02-27
~~~~~~~~~~~~~~~~~~
* API stabilised and v1 is now LTS
* ``gSeaGen`` reader removed, only supported until v0.29.2
Version 0
---------
0.29.2 / 2022-12-21
~~~~~~~~~~~~~~~~~~~
* Hotfix for AwkwardArray 2.0 incompatibility, now restricted to
be <2.0
0.29.1 / 2022-12-01
~~~~~~~~~~~~~~~~~~~
* Added ``codemeta.json`` to the MANIFEST
0.29.0 / 2022-11-07
~~~~~~~~~~~~~~~~~~~
* Update km3net-definitions to 3.0.0
0.28.0 / 2022-11-02
~~~~~~~~~~~~~~~~~~~
* Update km3net-definitions to 2.2.0-16-gbef370c
0.27.3 / 2022-10-20
~~~~~~~~~~~~~~~~~~~
* Fixed ``km3io.tools.fitinf()`` which always returned the 0th element
0.27.2 / 2022-10-04
~~~~~~~~~~~~~~~~~~~
* Added dockerisation
0.27.1 / 2022-09-20
~~~~~~~~~~~~~~~~~~~
* Fixes the issue where files procued with newer Jpp v17+ versions
errored due to a `header_uuid[16]` field
0.27.0 / 2022-07-20
~~~~~~~~~~~~~~~~~~~
* Adds the TimeConverter class to ``src/km3io/tools.py``
* Update km3net-dataformat requirement to version 0.3.6 or higher
* Update Black requirement to version 22.3.0 or higher, to prevent ``ImportError: cannot import name '_unicodefun' from 'click'``
* Remove ``requirements`` folder (all requirements are now configured in ``setup.cfg``)
0.26.1 / 2022-07-06
~~~~~~~~~~~~~~~~~~~
* The warning from OpenMP/Numba is now silenced
0.26.0 / 2022-06-27
~~~~~~~~~~~~~~~~~~~
* Added ``km3io.tools.is_nanobeacon()`` to check if the nanobeacon trigger bit is set
* Added ``km3io.tools.get_w2list_idx()`` to get the w2list index according to the
simulation program
0.25.2 / 2022-03-27
~~~~~~~~~~~~~~~~~~~
* Fixes the version
0.25.1 / 2022-03-24
~~~~~~~~~~~~~~~~~~~
* the ``.counter`` field for ``mc_trks/mc_tracks`` is now accessible
0.25.0 / 2022-03-14
~~~~~~~~~~~~~~~~~~~
* uproot 4.2.2+ required, which fixes a regression problem when reading doubly nested
structures
* Added a new, high-performance Summaryslice reader ``km3io.online.SummarysliceReader``
* The old ``km3io.OnlineReader.summarslices`` is now using the new ``SummarysliceReader``
which has a slightly different API (but at least an order of magnitude better
performance and much nicer high-level API thanks to AwkwardArrays)
0.24.1 / 2021-11-05
~~~~~~~~~~~~~~~~~~~
* The ``km3io.tools.is_bit_set()`` and all the related trigger mask checkers
(``is_3dmuon()``...) are now compatible with Numba
0.24.0 / 2021-11-02
~~~~~~~~~~~~~~~~~~~
* The field ``.a`` (amplitude) for Hits is now accessible
0.23.1 / 2021-09-28
~~~~~~~~~~~~~~~~~~~
* KM3NeT Dataformat definition updated to 2.1.0+
0.23.0 / 2021-07-03
~~~~~~~~~~~~~~~~~~~
* ``km3io.acoustics`` was added which provides ``RawAcousticsReader`` to
read -- wait for it -- raw acoustics data
0.22.0 / 2021-06-15
~~~~~~~~~~~~~~~~~~~
* Added ``km3io.tools.is_bit_set()`` along with some special methods to check
if a given ``trigger_mask`` (of an event or a hit) has a specific trigger
bit set, via ``km3io.tools.is_3dmuon``, ``km3io.tools.is_3dshower`` and
``km3io.tools.is_mxshower``
0.21.0 / 2021-04-08
~~~~~~~~~~~~~~~~~~~
* ``km3net-dataformat`` updated to v2.0.0-9-gbae3720
* mother ID and status are now read out for MC tracks
0.20.0 / 2021-02-18
~~~~~~~~~~~~~~~~~~~
* The fields ``.tdc``, ``.pos_{xyz}`` and ``.dir_{xyz}`` in ``.hits`` are
now read by default.
0.19.6 / 2021-02-01
~~~~~~~~~~~~~~~~~~~
* Improved header readout
0.19.5 / 2021-02-01
~~~~~~~~~~~~~~~~~~~
* Adds access to ``mc_event_time``
0.19.4 / 2021-02-01
~~~~~~~~~~~~~~~~~~~
* Fixed parsing error when a MC header contains invalid attribute names.
0.19.3 / 2020-12-17
~~~~~~~~~~~~~~~~~~~
* Added ``Branch.arrays()`` for high-level access of ``uproot.TBranch.arrays()``
0.19.2 / 2020-12-15
~~~~~~~~~~~~~~~~~~~
* Suppress FutureWarnings from uproot3
0.19.1 / 2020-12-11
~~~~~~~~~~~~~~~~~~~
* Minor hotfixes and cosmetics
0.19.0 / 2020-12-11
~~~~~~~~~~~~~~~~~~~
* Major update, coming closer to v1.0
* Now everything but the online-file access is based on uproot4 and awkward1
* Contact us if you encounter any problem after upgrading!
0.18.1 / 2020-12-04
~~~~~~~~~~~~~~~~~~~
* Fixed imports due to the rename of uproot to uproot3, uproot4 to uproot,
awkward to awkward0 and awkward1 to awkward
* Notice: the ``best_track*()`` functions are currently broken due to changes in
awkward which has not been fixed yet
0.18.0 / 2020-11-12
~~~~~~~~~~~~~~~~~~~
* A new tool ``km3io.tools.is_cc()`` has been added which can be used to
check if the events are of type CC
0.17.1 / 2020-10-19
~~~~~~~~~~~~~~~~~~~
* Requires ``awkward1>=0.3.1`` from now on (fixes an array-shape mismatch bug)
0.17.0 / 2020-10-13
~~~~~~~~~~~~~~~~~~~
* Final ;) ``km3io.tools.best_track`` implementation which provides
many different ways to chose the one and only "best track".
* Similar to ``km3net-dataformat/scripts/reconstruction.hh``, the
following functions can be used to retrieve the best track according
to the "standard definitions": ``km3io.tools.best_jmuon``, ``best_jshower``,
``best_dusjshower`` and ``best_aashower``
0.16.2 / 2020-10-07
~~~~~~~~~~~~~~~~~~~
* Adds ``.uuid`` attributes to ``OfflineReader`` and ``OnlineReader``
0.16.1 / 2020-09-30
~~~~~~~~~~~~~~~~~~~
* Fixed a bug in ``Branch.is_single``
0.16.0 / 2020-09-30
~~~~~~~~~~~~~~~~~~~
* Fixed the inconsistency of ``len()`` of mapped branches
See https://git.km3net.de/km3py/km3io/-/issues/39#note_18429
* Introduced ``Branch.is_single`` to check if a single branch is
selected
0.15.5 / 2020-09-30
~~~~~~~~~~~~~~~~~~~
* Fixed a tiny bug in ``km3io.tools.best_track``
0.15.4 / 2020-09-30
~~~~~~~~~~~~~~~~~~~
* Improved ``km3io.tools.best_track`` which now works nicely
when passing events and improves the error reporting
* ``tracks.usr`` is now hidden (again) from the user
0.15.3 / 2020-09-25
~~~~~~~~~~~~~~~~~~~
* Updated KM3NeT definitions to v1.2.4
0.15.2 / 2020-09-23
~~~~~~~~~~~~~~~~~~~
* Fixed a bug where the last bit of HRV or FIFO were incorrectly
masked when using ``km3io.online.get_channel_flags``
0.15.1 / 2020-07-15
~~~~~~~~~~~~~~~~~~~
* Added wheel packages for faster installation
0.15.0 / 2020-05-22
~~~~~~~~~~~~~~~~~~~
* Added reverse maps for index lookup of definitions
``km3io.definitions.fitparameters_idx`` etc.
0.14.2 / 2020-05-21
~~~~~~~~~~~~~~~~~~~
* Improved caching for awkward arrays in pumps
0.14.1 / 2020-05-21
~~~~~~~~~~~~~~~~~~~
* Improved caching for awkward arrays
0.14.0 / 2020-04-29
~~~~~~~~~~~~~~~~~~~
* ``events.mc_tracks.usr`` and ``events.mc_tracks.usr_names`` are now
correctly parsed
0.13.0 / 2020-04-26
~~~~~~~~~~~~~~~~~~~
* ``km3io.tools.unique`` and ``km3io.tools.uniquecount`` were added to help
working with unique elements (e.g. DOM IDs or channel IDs)
* Internal restructuring of ``.tools``, ROOT related stuff is moved
to ``.rootio``
0.12.0 / 2020-04-26
~~~~~~~~~~~~~~~~~~~
* Added ``.close()`` to the Offline and Online reader classes
* The Offline and Online reader classes now support context managers
(``with km3io.OfflineReader(filename) as r: ...``)
0.11.0 / 2020-04-19
~~~~~~~~~~~~~~~~~~~
* DAQ was renamed to online
* Several improviements, bugfixes etc.
0.10.0 / 2020-04-01
~~~~~~~~~~~~~~~~~~~
* The offline I/O has been refactored and now supports slicing à la numpy
0.9.1 / 2020-03-29
~~~~~~~~~~~~~~~~~~
* Added support for gSeaGen files
0.9.0 / 2020-03-03
~~~~~~~~~~~~~~~~~~
* Added support for the ``usr`` field of events
0.8.3 / 2020-02-25
~~~~~~~~~~~~~~~~~~
* The times of snapshot and triggered hits were parsed as big endian (standard)
ROOT endianness, however, Jpp stores that as little endian with a custom
streamer. This is now fixed...
0.8.2 / 2020-02-14
~~~~~~~~~~~~~~~~~~
* minor fixes
0.8.1 / 2020-02-10
~~~~~~~~~~~~~~~~~~
* update of reco data from offline files
* Documentation on how to read DAQ data
0.8.0 / 2020-01-23
~~~~~~~~~~~~~~~~~~
* Offline file headers are now accessible
0.7.0 / 2020-01-23
~~~~~~~~~~~~~~~~~~
* Reading of summary slice status information is now supported
0.6.3 / 2020-01-09
~~~~~~~~~~~~~~~~~~
* Bugfixes
0.6.2 / 2019-12-22
~~~~~~~~~~~~~~~~~~
* Fixes slicing of ``OfflineTracks``
0.6.1 / 2019-12-21
~~~~~~~~~~~~~~~~~~
* Minor cleanup
0.6.0 / 2019-12-21
~~~~~~~~~~~~~~~~~~
* Jpp things were renamed to DAQ things (;
* Reading of summary slices is done!
0.5.1 / 2019-12-18
~~~~~~~~~~~~~~~~~~
* Cosmetics
0.5.0 / 2019-12-16
~~~~~~~~~~~~~~~~~~
* Massive update of the aanet data format reader
0.4.0 / 2019-11-22
~~~~~~~~~~~~~~~~~~~
* First timeslice frame readout prototype
0.3.0 / 2019-11-19
~~~~~~~~~~~~~~~~~~~
* Preliminary Jpp timeslice reader prototype
* Updated ``AanetReader``
* Updated docs
0.2.1 / 2019-11-15
~~~~~~~~~~~~~~~~~~~
* Updated docs
0.2.0 / 2019-11-15
~~~~~~~~~~~~~~~~~~~
* ``JppReader`` added, which is able to read events!
0.1.0 / 2019-11-15
~~~~~~~~~~~~~~~~~~~
* First release
* Prototype implementation of the ``AanetReader``
Contact Us
----------
Join the KM3Pipe channel here: https://chat.km3net.de/channel/km3pipe
Filing Bugs or Feature Requests
-------------------------------
Please **always** create an issue when you encounter any bugs, problems or
need a new feature. Emails and private messages are not meant to communicate
such things!
Use the appropriate template and file a new issue here:
https://git.km3net.de/km3py/km3io/issues
You can browse all the issues here: https://git.km3net.de/km3py/km3io/-/issues
Please follow the instructions in the templates to provide all the
necessary information which will help other people to understand the
situation.
Improve km3io
---------------
Check out our KanBan board https://git.km3net.de/km3py/km3io/-/boards,
which shows all the open issues in three columns:
- *discussion*: The issues which are yet to be discussed (e.g. not clear how to proceed)
- *todo*: Issues tagged with this label are ready to be tackled
- *doing*: These issues are currently "work in progress". They can however be
put tossed back to *todo* column at any time if the development is suspended.
Here is the recommended workflow if you want to improve km3io. This is a
standard procedure for collaborative software development, nothing exotic!
Feel free to contribute ;)
Make a Fork of km3io
~~~~~~~~~~~~~~~~~~~~~~
You create a fork (your full own copy of the
repository), change the code and when you are happy with the changes, you create
a merge request, so we can review, discuss and add your contribution.
Merge requests are automatically tested on our GitLab CI server and you
don't have to do anything special.
Go to http://git.km3net.de/km3py/km3io and click on "Fork".
After that, you will have a full copy of km3io with write access under an URL
like this: ``http://git.km3net.de/your_git_username/km3io``
Clone your Fork to your PC
~~~~~~~~~~~~~~~~~~~~~~~~~~
Get a local copy to work on (use the SSH address `git@git...`, not the HTTP one)::
git clone git@git.km3net.de:your_git_username/km3io.git
Now you need to add a reference to the original repository, so you can sync your
own fork with the km3io repository::
cd km3io
git remote add upstream git@git.km3net.de:km3py/km3io.git
Keep your Fork Up to Date
~~~~~~~~~~~~~~~~~~~~~~~~~
To get the most recent commits (including all branches), run::
git fetch upstream
This will download all the missing commits and branches which are now accessible
using the ``upstream/...`` prefix::
$ git fetch upstream
From git.km3net.de:km3py/km3io
* [new branch] gitlab_jenkins_ci_test -> upstream/gitlab_jenkins_ci_test
* [new branch] legacy -> upstream/legacy
* [new branch] master -> upstream/master
If you want to update for example your **own** ``master`` branch
to contain all the changes on the official ``master`` branch of km3io,
switch to it first with::
git checkout master
and then merge the ``upstream/master`` into it::
git merge upstream/master
Make sure to regularly ``git fetch upstream`` and merge changes to your own branches.
Push your changes to Gitlab regularly
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Make sure to keep your fork up to date on the GitLab server by pushing
all your commits regularly using::
git push
Install in Developer Mode
~~~~~~~~~~~~~~~~~~~~~~~~~
km3io can be installed in ``dev-mode``, which means, it links itself to your
site-packages and you can edit the sources and test them without the need
to reinstall km3io all the time. Although you will need to restart any
``python``, ``ipython`` or ``jupyter``-notebook (only the kernel!) if you
imported km3io before you made the changes.
Go to your own fork folder (as described above) and check out the branch you
want to work on::
git checkout master # the main development branch (should always be stable)
make install-dev
Running the Test Suite
~~~~~~~~~~~~~~~~~~~~~~
Make sure to run the test suite first to see if everything is working
correctly::
$ make test
This should give you a green bar, with an output like this::
$ make test
py.test --junitxml=./reports/junit.xml -o junit_suite_name=km3io tests
=========================================== test session starts ===========================================
platform linux -- Python 3.8.5, pytest-5.3.5, py-1.8.1, pluggy-0.13.1
rootdir: /home/tgal/Dev/km3io
plugins: flake8-1.0.4, pylint-0.15.0, cov-2.8.1
collected 126 items
tests/test_gseagen.py ..... [ 3%]
tests/test_km3io.py . [ 4%]
tests/test_offline.py ................................s............. [ 41%]
tests/test_online.py ................................... [ 69%]
tests/test_tools.py ....................................... [100%]
----------------------- generated xml file: /home/tgal/Dev/km3io/reports/junit.xml ------------------------
=============================== 125 passed, 1 skipped, 4 warnings in 6.54s ================================
Run the tests every time you make changes to see if you broke anything! It usually
takes just a few seconds and ensures that you don't break existing code. It's
also an easy way to spot syntax errors ;)
You can also start a script which will watch for file changes and retrigger
a test suite run every time for you. It's a nice practice to have a terminal
open running this script to check your test results continuously::
make test-loop
Time to Code
~~~~~~~~~~~~
We develop new features and fix bugs on separate branches and merge them
back to ``master`` when they are stable. Merge requests (see below) are also
pointing towards this branch.
If you are working on your own fork, you can stay on your own ``master`` branch
and create merge requests from that.
Code Style
~~~~~~~~~~
Make sure to run ``black`` over the code, which ensures that the code style
matches the one we love and respect. We have a tool which makes it easy::
make black
Create a Merge Request (aka Pull Request)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Go to https://git.km3net.de/km3py/km3io/merge_requests/new and select
your source branch, which contains the changes you want to be included in km3io
and select the ``master`` branch as target branch.
That's it, the merge will be accepted if everything is OK ;)
If you want to join the km3io dev-team, let us know!:)
FROM python:3.12.7-bookworm
LABEL maintainer="Tamas Gal <tgal@km3net.de>"
ENV INSTALL_DIR /km3io
RUN apt-get update
RUN apt-get install -y -qq git python3-pip && apt-get -y clean && apt-get autoclean
RUN python3 -m pip install --upgrade pip setuptools wheel
ADD . $INSTALL_DIR
RUN cd $INSTALL_DIR && python3 -m pip install . ".[dev]" ".[extras]"
# Clean up
RUN cd / && rm -rf $INSTALL_DIR
RUN rm -rf /var/cache
include README.rst CHANGELOG.rst CONTRIBUTING.rst
include LICENSE
include setup.py setup.cfg pyproject.toml
include src/km3io/version.py
include codemeta.json
recursive-include tests
......@@ -8,21 +8,21 @@ install:
pip install .
install-dev:
pip install -r requirements-dev.txt
pip install -e .
pip install -e ".[dev]"
python -m ipykernel install --user --name=km3io
clean:
python setup.py clean --all
test:
py.test --junitxml=./reports/junit.xml -o junit_suite_name=$(PKGNAME) $(PKGNAME)
py.test --junitxml=./reports/junit.xml -o junit_suite_name=$(PKGNAME) tests
test-cov:
py.test --cov ./ --cov-report term-missing --cov-report xml:reports/coverage.xml --cov-report html:reports/coverage $(PKGNAME)
py.test --cov src/km3io --cov-report term-missing --cov-report xml:reports/coverage.xml --cov-report html:reports/coverage tests
test-loop:
py.test $(PKGNAME)
ptw --ext=.py,.pyx --ignore=doc $(PKGNAME)
py.test tests
ptw --ext=.py,.pyx --ignore=doc tests
flake8:
py.test --flake8
......@@ -35,12 +35,21 @@ docstyle:
lint:
py.test --pylint
dependencies:
pip install -Ur requirements.txt
.PHONY: black
black:
black --exclude '/_definitions/|version.py' src/km3io
black examples
black tests
black doc/conf.py
black setup.py
.PHONY: yapf
yapf:
yapf -i -r $(PKGNAME)
yapf -i setup.py
.PHONY: black-check
black-check:
black --check --exclude '/_definitions/|version.py' src/km3io
black --check examples
black --check tests
black --check doc/conf.py
black --check setup.py
.PHONY: all clean install install-dev test test-nocov flake8 pep8 dependencies docstyle
.PHONY: all clean install install-dev test test-nocov flake8 pep8 docstyle black black-check
The km3io Python package
========================
.. image:: https://git.km3net.de/km3py/km3io/badges/master/pipeline.svg
:target: https://git.km3net.de/km3py/km3io/pipelines
.. image:: https://git.km3net.de/km3py/km3io/badges/master/coverage.svg
:target: https://km3py.pages.km3net.de/km3io/coverage
.. image:: https://git.km3net.de/examples/km3badges/-/raw/master/docs-latest-brightgreen.svg
:target: https://km3py.pages.km3net.de/km3io
.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.7382620.svg
:target: https://doi.org/10.5281/zenodo.7382620
This software provides a set of Python classes to read KM3NeT ROOT files
without having ROOT, Jpp or aanet installed. It only depends on Python 3.5+ and the amazing `uproot <https://github.com/scikit-hep/uproot>`__ package and gives you access to the data via `numpy <https://www.numpy.org>`__ and `awkward <https://awkward-array.readthedocs.io>`__ arrays.
It's very easy to use and according to the `uproot <https://github.com/scikit-hep/uproot>`__ benchmarks, it is able to outperform the original ROOT I/O performance.
Installation
============
Install km3io using pip::
pip install km3io
or conda::
conda install km3io
To get the latest (stable) development release::
pip install git+https://git.km3net.de/km3py/km3io.git
Docker::
docker run -it docker.km3net.de/km3io
Singularity::
wget https://sftp.km3net.de/singularity/km3io_v0.27.2.sif # pick the version you like
singularity shell km3io_v0.27.2.sif
**Reminder:** km3io is **not** dependent on aanet, ROOT or Jpp!
Questions
=========
If you have a question about km3io, please proceed as follows:
- Read the documentation below.
- Explore the `examples <https://km3py.pages.km3net.de/km3io/examples.html>`__ in the documentation.
- Haven't you found an answer to your question in the documentation, post a git issue with your question showing us an example of what you have tried first, and what you would like to do.
- Have you noticed a bug, please post it in a git issue, we appreciate your contribution.
Introduction
------------
Most of km3net data is stored in root files. These root files are created using
the `KM3NeT Dataformat library
<https://git.km3net.de/common/km3net-dataformat>`__ A ROOT file created with
`Jpp <https://git.km3net.de/common/jpp>`__ is an "online" file and all other
software usually produces "offline" files.
km3io is a Python package that provides access to offline files with its
``OfflineReader`` class and a special one to read gSeaGen files. All of these
ROOT files can be read without installing any other software like Jpp, aanet or
ROOT. km3io v1.1 and earlier also support the access to online files (events,
summaryslices and timeslices). This feature has been dropped due to a lack of
mainteinance power and inf favour of the `KM3io.jl <https://git.km3net.de/common/KM3io.jl>`__` Julia Package, which
provides high-performances access to all ROOT files and should also be
prioritised over ``km3io`` when performance matters (which does, most of the
time).
Data in km3io is returned as ``awkward.Array`` which is an advance Numpy-like container type to store
contiguous data for high performance computations.
Such an ``awkward.Array`` supports any level of nested arrays and records which can have different lengths, in contrast to Numpy where everything has to be rectangular.
The example is shown below shows the array which contains the ``dir_z`` values
of each track of the first 4 events. The type ``4 * var * float64`` means that
it has 4 subarrays with variable lengths of type ``float64``:
.. code-block:: python3
>>> import km3io
>>> from km3net_testdata import data_path
>>> f = km3io.OfflineReader(data_path("offline/numucc.root"))
>>> f[:4].tracks.dir_z
<Array [[0.213, 0.213, ... 0.229, 0.323]] type='4 * var * float64'>
The same concept applies to all other branches, including ``hits``, ``mc_hits``,
``mc_tracks``, ``t_sec`` etc.
Architecture overview
---------------------
``km3io`` utilises ``uproot`` behind the scenes and creates a lazy and thin
wrapper which offers convenient slicing and iterations by delaying the access to
the actual ROOT data branches to the very last moment. When using the iteration
functionality, the data is loaded in chunks and the iteration is done over e.g.
events in each chunk or a bunch of frames in case of the summaryslice reader.
The base class for the event-based readout is the ``km3io.rootio.EventReader``
class. When subclassing this class, the branches, aliases and nested branches
need to be defined in the static variables which are then used to mask unwanted
attributes. Especially in case of the Offline ROOT format, where the "one class
fits all" design was chosen, it is distracting that e.g. a `Hit` has many
attributes which make no sense depending on the context (MC hit, raw hit etc.).
By specifing the branches explicitely, the user API will only expose the
meaningful fields.
The online ROOT format support is partly still based on ``uproot3``.
Many of the utility functions are using Numba to achieve the best possible
performance. ``km3io`` does not offer alternative implementations, so Numba is a
strict dependency and an integral part of the implementation.
Offline files reader
--------------------
In general an offline file has two attributes to access data: the header and the events. Let's start with the header.
Reading the file header
"""""""""""""""""""""""
To read an offline file start with opening it with the ``OfflineReader``:
.. code-block:: python3
>>> import km3io
>>> from km3net_testdata import data_path
>>> f = km3io.OfflineReader(data_path("offline/numucc.root"))
Accessing is as easy as typing:
.. code-block:: python3
>>> f.header
<km3io.offline.Header at 0x7fcd81025990>
Printing it will give an overview of the structure:
.. code-block:: python3
>>> print(f.header)
MC Header:
DAQ(livetime=394)
PDF(i1=4, i2=58)
can(zmin=0, zmax=1027, r=888.4)
can_user: can_user(field_0=0.0, field_1=1027.0, field_2=888.4)
coord_origin(x=0, y=0, z=0)
cut_in(Emin=0, Emax=0, cosTmin=0, cosTmax=0)
cut_nu(Emin=100, Emax=100000000.0, cosTmin=-1, cosTmax=1)
cut_primary(Emin=0, Emax=0, cosTmin=0, cosTmax=0)
cut_seamuon(Emin=0, Emax=0, cosTmin=0, cosTmax=0)
decay: decay(field_0='doesnt', field_1='happen')
detector: NOT
drawing: Volume
genhencut(gDir=2000, Emin=0)
genvol(zmin=0, zmax=1027, r=888.4, volume=2649000000.0, numberOfEvents=100000)
kcut: 2
livetime(numberOfSeconds=0, errorOfSeconds=0)
model(interaction=1, muon=2, scattering=0, numberOfEnergyBins=1, field_4=12)
ngen: 100000.0
norma(primaryFlux=0, numberOfPrimaries=0)
nuflux: nuflux(field_0=0, field_1=3, field_2=0, field_3=0.5, field_4=0.0, field_5=1.0, field_6=3.0)
physics(program='GENHEN', version='7.2-220514', date=181116, time=1138)
seed(program='GENHEN', level=3, iseed=305765867, field_3=0, field_4=0)
simul(program='JSirene', version=11012, date='11/17/18', time=7)
sourcemode: diffuse
spectrum(alpha=-1.4)
start_run(run_id=1)
target: isoscalar
usedetfile: false
xlat_user: 0.63297
xparam: OFF
zed_user: zed_user(field_0=0.0, field_1=3450.0)
To read the values in the header one can call them directly, as the structures
are simple ``namedtuple``-like objects:
.. code-block:: python3
>>> f.header.DAQ.livetime
394
>>> f.header.cut_nu.Emin
100
>>> f.header.genvol.numberOfEvents
100000
Reading offline events
""""""""""""""""""""""
Events are at the top level of an offline file, so that each branch of an event
is directly accessible at the ``OfflineReader`` instance. The ``.keys()`` method
can be used to list the available attributes. Notice that some of them are aliases
for backwards compatibility (like ``mc_tracks`` and ``mc_trks``). Another
backwards compatibility feature is the ``f.events`` attribute which is simply
mapping everything to ``f``, so that ``f.events.mc_tracks`` is the same as
``f.mc_tracks``.
.. code-block:: python3
>>> f
OfflineReader (10 events)
>>> f.keys()
{'comment', 'det_id', 'flags', 'frame_index', 'hits', 'id', 'index',
'mc_hits', 'mc_id', 'mc_run_id', 'mc_t', 'mc_tracks', 'mc_trks',
'n_hits', 'n_mc_hits', 'n_mc_tracks', 'n_mc_trks', 'n_tracks',
'n_trks', 'overlays', 'run_id', 't_ns', 't_sec', 'tracks',
'trigger_counter', 'trigger_mask', 'trks', 'usr', 'usr_names',
'w', 'w2list', 'w3list'}
>>> f.tracks
<Branch [10] path='trks'>
>>> f.events.tracks
<Branch [10] path='trks'>
The ``[10]`` denotes that there are ``10`` events available, each containing a sub-array of ``tracks``.
Using <TAB> completion gives an overview of available data. Alternatively the attribute `fields`
can be used on event-branches and to see what is available for reading.
.. code-block:: python3
>>> f.tracks.fields
['id',
'pos_x',
'pos_y',
'pos_z',
'dir_x',
'dir_y',
'dir_z',
't',
'E',
'len',
'lik',
'rec_type',
'rec_stages',
'fitinf']
Reading the reconstructed values like energy and direction of an event can be done with:
.. code-block:: python3
>>> f.events.tracks.E
<Array [[117, 117, 0, 0, 0, ... 0, 0, 0, 0, 0]] type='10 * var * float64'>
The ``Array`` in this case is an `awkward <https://awkward-array.readthedocs.io>`__ array with the data type
``10 * var * float64`` which means that there are ``10`` sub-arrays with ``var``iable lengths of type ``float64``.
Awkward arrays allow high-performance access to arrays which are not rectangular (in contrast to ``numpy``).
Read the documention of AwkwardArray to learn how to work with these structures efficiently. One example
to retrieve the energy of the very first reconstructed track for the first three events is:
.. code-block:: python3
>>> f.events.tracks.E[:3, 0]
<Array [117, 4.4e+03, 8.37] type='3 * float64'>
Online files reader
-------------------
The support to read online ROOT files has been dropped in ``km3io`` v1.2.
{
"@context": "https://doi.org/10.5063/schema/codemeta-2.0",
"@type": "SoftwareSourceCode",
"license": "https://spdx.org/licenses/MIT",
"codeRepository": "git+https://git.km3net.de/km3py/km3io.git",
"contIntegration": "https://git.km3net.de/km3py/km3io/-/pipelines",
"dateCreated": "2019-11-15",
"datePublished": "2019-11-15",
"dateModified": "2023-06-15",
"downloadUrl": "https://git.km3net.de/km3py/km3io",
"issueTracker": "https://git.km3net.de/km3py/km3io/-/issues",
"name": "km3io",
"softwareVersion": "1.0.0",
"maintainer": "Tamas Gal <tamas.gal@fau.de>",
"readme": "https://git.km3net.de/km3py/km3io/-/blob/master/README.rst",
"description": "Pure Python I/O library for KM3NeT related files.",
"applicationCategory": "Astroparticle physics",
"developmentStatus": "active",
"referencePublication": "https://doi.org/10.5281/zenodo.808829",
"releaseNotes": "https://km3py.pages.km3net.de/km3io/changelog.html",
"copyrightHolder": "KM3NeT Collaboration",
"keywords": [
"i/o",
"astroparticle physics",
"km3net"
],
"programmingLanguage": [
"Python 3"
],
"operatingSystem": [
"Linux",
"macOS",
"Windows"
],
"relatedLink": [
"https://km3py.pages.km3net.de/km3io/"
],
"author": [
{
"@type": "Person",
"@id": "https://orcid.org/0000-0001-7821-8673",
"givenName": "Tamas",
"familyName": "Gal",
"email": "tamas.gal@fau.de",
"affiliation": {
"@type": "Organization",
"name": "Friedrich-Alexander-University Erlangen-Nuremberg, Erlangen Centre for Astroparticle Physics"
}
}
],
"contributor": [
{
"@type": "Person",
"@id": "https://orcid.org/0000-0003-3722-086X",
"givenName": "Johannes",
"familyName": "Schumann",
"affiliation": {
"@type": "Organization",
"name": "Friedrich-Alexander-University Erlangen-Nuremberg, Erlangen Centre for Astroparticle Physics"
}
},
{
"@type": "Person",
"@id": "https://orcid.org/0000-0002-5593-2580",
"givenName": "Zineb",
"familyName": "Aly"
}
]
}
.rst-content .highlight>pre {
line-height: 18px;
}
doc/_static/default_gallery_thumbnail.png

23.7 KiB

Changelog
=========
.. include:: ../CHANGELOG.rst
......@@ -13,14 +13,26 @@
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import sys
import os
from datetime import date
import sphinx_rtd_theme
from pkg_resources import get_distribution
# -- Project information -----------------------------------------------------
try:
from importlib.metadata import version as get_version
version = get_version("km3io")
except ImportError:
from pkg_resources import get_distribution
project = 'km3io'
copyright = '2019, Zineb Aly, Tamas Gal, Johannes Schumann'
author = 'Zineb Aly, Tamas Gal, Johannes Schumann'
version = get_distribution("km3io").version
short_version = ".".join(version.split(".")[:2])
project = "km3io {}".format(short_version)
copyright = "{0}, Zineb Aly and Tamas Gal".format(date.today().year)
author = "Zineb Aly, Tamas Gal, Johannes Schumann"
# -- General configuration ---------------------------------------------------
......@@ -28,25 +40,47 @@ author = 'Zineb Aly, Tamas Gal, Johannes Schumann'
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
"sphinx.ext.autodoc",
"sphinx.ext.autosummary",
"sphinx.ext.viewcode",
"autoapi.extension",
"numpydoc",
"sphinx_gallery.gen_gallery",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# AutoAPI
autoapi_type = "python"
autoapi_dirs = ["../src/km3io"]
autoapi_options = ["members", "undoc-members", "show-module-summary"]
autoapi_include_summaries = True
# Gallery
sphinx_gallery_conf = {
"backreferences_dir": "modules/generated",
"default_thumb_file": "_static/default_gallery_thumbnail.png",
"examples_dirs": "../examples", # path to your example scripts
"gallery_dirs": "auto_examples", # path to where to save gallery generated output
"show_memory": True,
}
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
html_title = "km3io {}".format(version)
Contribute
==========
.. include:: ../CONTRIBUTING.rst
========
Examples
========
.. include:: auto_examples/index.rst
......@@ -3,13 +3,24 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to km3io's documentation!
=====================================
.. include:: ../README.rst
.. toctree::
:maxdepth: 2
:caption: Contents:
examples
contribute
changelog
autoapi/index
autoapi/km3io/aanet/index
autoapi/km3io/jpp/index
Code Coverage <http://km3py.pages.km3net.de/km3io/coverage>
Source (Git) <https://git.km3net.de/km3py/km3io.git>
Indices and tables
......
File moved
Reading offline data
====================
Feel free to explore and extend our examples!
"""
Reading Offline events
======================
The following example shows how to access events data in an offline ROOT file, which is
written by aanet software.
Note: the offline file used here has MC offline data and was intentionaly reduced
to 10 events.
"""
import km3io as ki
from km3net_testdata import data_path
#####################################################
# First, pass a filename to the `OfflineReader` class to open the file.
# Note that only some meta information is read into memory.
r = ki.OfflineReader(data_path("offline/numucc.root"))
#####################################################
# Accessing the file header
# -------------------------
# Note that not all file headers are supported, so don't be surprised if
# nothing is returned when the file header is called (this can happen if your file
# was produced with old versions of aanet).
h = r.header
print(h)
#####################################################
# Accessing the events data
# -------------------------
# Note that not all data is loaded in memory (again), so printing
# events will only return how many events were found in the file.
print(r.events)
#####################################################
# to explore the events keys:
keys = r.events.keys()
print(keys)
#####################################################
# to access the number of hits associated with each event:
n_hits = r.events.n_hits
print(n_hits)
#####################################################
# to access the number of tracks associated with each event:
n_tracks = r.events.n_tracks
print(n_tracks)
#####################################################
# to access the number of mc hits associated with each event:
n_mc_hits = r.events.n_mc_hits
print(n_mc_hits)
#####################################################
# to access the number of mc tracks associated with each event:
n_mc_tracks = r.events.n_mc_tracks
print(n_mc_tracks)
#####################################################
# to access the overlays:
overlays = r.events.overlays
print(overlays)
#####################################################
# That's it! you can access any key of your interest in the events
# keys in the exact same way.
#####################################################
# item selection in events data
# -----------------------------
# events can be selected as you would select an item from a numpy array.
# for example, to select the mc_hits in event 0:
print(r.events[0].mc_hits)
#####################################################
# or:
print(r.events.mc_hits[0])
#####################################################
# slicing of events
# -----------------
# you can select a slice of events data. For example, to select the number
# of mc hits in the first 5 events:
print(r.events.n_mc_hits[0:5])
#####################################################
# or:
print(r.events[0:5].n_mc_hits)
#####################################################
# you can apply masks to events data as you would do with numpy arrays.
# For example, to select the number of hits higher than 50:
mask = r.events.n_mc_hits > 50
print(r.events.n_mc_hits[mask])
#####################################################
# or:
print(r.events.n_mc_tracks[mask])