First release

Project.toml

@@ -3,6 +3,9 @@ uuid = "a0704f48-f225-4b05-83f3-c70968ad2d8c"
 authors = ["Tamas Gal"]
 version = "0.1.0"
 
+[deps]
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
 [compat]
 julia = "1"
 

README.md

@@ -5,7 +5,9 @@
 [![Build Status](https://git.km3net.de/common/KM3Aux.jl/badges/main/pipeline.svg)](https://git.km3net.de/common/KM3Aux.jl/pipelines)
 [![Coverage](https://git.km3net.de/common/KM3Aux.jl/badges/main/coverage.svg)](https://git.km3net.de/common/KM3Aux.jl/commits/main)
 
-Welcome to the `KM3Aux.jl` repository!
+Welcome to the `KM3Aux.jl` repository! Your partner for accessing files from the
+legendary [KM3NeT Auxiliary Files Archive](https://git.km3net.de/auxiliary_data/calibration)
+for calibration data.
 
 
 ## Documentation
@@ -31,6 +33,17 @@ After that, you can add `KM3Aux.jl` just like any other Julia package:
 
 ## Quickstart
 
+Make sure to clone the [KM3NeT Auxiliary Files Archive](https://git.km3net.de/auxiliary_data/calibration)
+to a folder and set the environment variable `KM3AUX_CALIB_PATH` to that path:
+
+    git clone git@git.km3net.de:auxiliary_data/calibration.git /path/to/the/archive
+    export KM3AUX_CALIB_PATH=/path/to/the/archive
+
 ``` julia-repl
 julia> using KM3Aux
+
+julia> fpath = filepath(160, 19468, "pmt")
+"/path/to/the/archive/00000160/pmt/_H_1.1.3/00019465/00019532/1.txt"
 ```
+
+

src/KM3Aux.jl

 module KM3Aux
 
-export meaningoflife
+using Printf
+
+export filepath
+
+
+function getcalibpath()
+    p = get(ENV, "KM3AUX_CALIB_PATH", "")
+    if p == ""
+        error(
+            "The environment variable 'KM3AUX_CALIB_PATH' is not set. " *
+            "It should point to a clone of the " *
+            "https://git.km3net.de/auxiliary_data/calibration repository."
+        )
+    end
+    if !ispath(p)
+        error(
+            "The KM3AUX_CALIB_PATH environment variable points to a " *
+            "non-existent directory."
+        )
+    end
+    if !isdir(p)
+        error(
+            "The KM3AUX_CALIB_PATH environment variable points to an " *
+            "invalid directory."
+        )
+    end
+    p
+end
+
+
+"""
+    filepath(detid::Int, runid::Int, type::AbstractString)
+
+Returns the filepath for an auxiliary file based on the detector ID, run and type.
+The environment variable `KM3AUX_CALIB_PATH` is used as base directory and the latest
+version of the file is picked. The auxiliary archive is using LFS and the file will
+be pulled automatically.
+
+The following version scheme is used in the archive:
+
+    ./XXXXXXXX/<type>/_<version>/YYYYYYYY/ZZZZZZZZ/<counter>.<extension>
+
+"""
+function filepath(detid::Int, runid::Int, type::AbstractString)
+    calibpath = getcalibpath()
+    detpath = joinpath(calibpath, zeropad8(detid))
+    !isdir(detpath) && error("No data found for detector with ID '$detid'.")
+
+    typepath = joinpath(detpath, type)
+    if !isdir(typepath)
+        available_types = join(readdir(detpath), ", ", " and ")
+        error("Unknown type '$type'. Available types: $(available_types).")
+    end
+
+    versions = readdir(typepath) |> versionsort
+    length(versions) == 0 && error("No data of type '$type' found for detector with ID '$detid'.")
+
+    v = versions[end]
+    versionpath = joinpath(typepath, v)
+
+    runs = sort([parse(Int, run) for run in readdir(versionpath)])
+    run_begin_idx = searchsortedlast(runs, runid)
+    run_begin_idx == 0 && error("No data of type '$type' found for detector with ID '$detid' and run '$runid'.")
+    run_begin = runs[run_begin_idx]
+
+    runpath = joinpath(versionpath, zeropad8(run_begin))
+    run_end = readdir(runpath)
+    length(run_end) == 0 && error("Incomplete data of type '$type' for detector with ID '$detid' and run '$runid'.")
+
+    auxpath = joinpath(runpath, run_end[end])  # run_end should only have one element
+    fnames = readdir(auxpath) |> versionsort
+    length(fnames) == 0 && error("Missing data of type '$type' for detector with ID '$detid' and run '$runid'.")
+
+    fpath = joinpath(auxpath, fnames[end])
+
+    !needs_lfs_pull(fpath) && return fpath
+
+    # pulling from LFS
+    lfspath = relpath(fpath, calibpath)
+    # wait=false to suppress output of the Git command
+    process = run(Cmd(`git lfs pull --exclude= --include $lfspath`, dir=calibpath, ignorestatus=true), wait=false)
+    # we need to wait for it...
+    wait(process)
+    if process.exitcode != 0
+        command = join(process.cmd.exec, " ")
+        error(
+            "Git LFS command failed with non-zero exit code ($(process.exitcode)).\n" *
+            "  Command: $command"
+        )
+    end
+
+    fpath
+end
 
 """
-    meaningoflife()
+    needs_lfs_pull(fpath::AbstractString)
+
+Checks if a given filepath is an LFS object yet to be pulled.
+"""
+function needs_lfs_pull(fpath::AbstractString)
+    s = String(read(fpath, 39))
+    s == "version https://git-lfs.github.com/spec"
+end
+
+
+"""
+    zeropad8(x::Integer)
+
+Creates an 8 characters long zeropadded string for a given number.
+
+# Examples
+
+```
+julia> KM3Aux.zeropad8(42)
+"00000042"
+"""
+zeropad8(x::Integer) = @sprintf "%08d" x
+
+
+"""
+    versionsort(v::Vector{T}) where T<:AbstractString
+
+Sorts a collection of strings based on version identifiers.
+"""
+function versionsort(v::Vector{T}) where T<:AbstractString
+    sort(v; by=expandversion)
+end
+
+
+"""
+    expandversion(s::AbstractString)
+
+Expands the version components into tuples, useful for sorting.
+
+# Examples
+
+```
+julia> KM3Aux.expandversion("_H_1.12.3")
+("H", 1, 12, 3)
 
-Determines the meaning of life without floating point precision.
+julia> KM3Aux.expandversion("42.datx")
+42
+```
 """
-function meaningoflife()
-    a = 21
-    b = 2
-    return a * b
+function expandversion(s::AbstractString)
+    m = match(r"_(\w)_(\d+)\.(\d+)\.(\d+)", s)
+    if !isnothing(m)
+        return(m[1], parse(Int, m[2]), parse(Int, m[3]), parse(Int, m[4]))
+    else
+        m = match(r"(\d+)\.?+", s)
+        if !isnothing(m)
+            return parse(Int, m[1])
+        end
+    end
+    return s
 end
 
 end

test/misc.jl

-using KM3Aux
-using Test
-
-
-@testset "meaningoflife()" begin
-    @test 42 == meaningoflife()
-end

test/runtests.jl

 using Test
+using KM3Aux
 
-include("misc.jl")
+@testset "getfilepath" begin
+    calibpath = joinpath(@__DIR__, "calibration")
+    ENV["KM3AUX_CALIB_PATH"] = calibpath
+
+    @test filepath(160, 19466, "pmt") == joinpath(calibpath, "00000160", "pmt", "_H_10.1.3", "00019465", "00019532", "10.txt")
+end