diff --git a/Project.toml b/Project.toml index 3422b6d43085c529b3ac87ad80061e3588388d73..640eaec5439c44ba08c3490ce9051d78611c2035 100644 --- a/Project.toml +++ b/Project.toml @@ -3,6 +3,9 @@ uuid = "a0704f48-f225-4b05-83f3-c70968ad2d8c" authors = ["Tamas Gal"] version = "0.1.0" +[deps] +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" + [compat] julia = "1" diff --git a/README.md b/README.md index 7efb32a86544397b15c90bc79ffa299da5f4da51..09c74452ea995a1e514c8d97e172cf799257b8dd 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ [](https://git.km3net.de/common/KM3Aux.jl/pipelines) [](https://git.km3net.de/common/KM3Aux.jl/commits/main) -Welcome to the `KM3Aux.jl` repository! +Welcome to the `KM3Aux.jl` repository! Your partner for accessing files from the +legendary [KM3NeT Auxiliary Files Archive](https://git.km3net.de/auxiliary_data/calibration) +for calibration data. ## Documentation @@ -31,6 +33,17 @@ After that, you can add `KM3Aux.jl` just like any other Julia package: ## Quickstart +Make sure to clone the [KM3NeT Auxiliary Files Archive](https://git.km3net.de/auxiliary_data/calibration) +to a folder and set the environment variable `KM3AUX_CALIB_PATH` to that path: + + git clone git@git.km3net.de:auxiliary_data/calibration.git /path/to/the/archive + export KM3AUX_CALIB_PATH=/path/to/the/archive + ``` julia-repl julia> using KM3Aux + +julia> fpath = filepath(160, 19468, "pmt") +"/path/to/the/archive/00000160/pmt/_H_1.1.3/00019465/00019532/1.txt" ``` + + diff --git a/src/KM3Aux.jl b/src/KM3Aux.jl index 764edcfc73bf7b0f14b5f6793a09bdeb7738feb3..d09ccc73ed42ce458955410982c16bc75a930c95 100644 --- a/src/KM3Aux.jl +++ b/src/KM3Aux.jl @@ -1,16 +1,160 @@ module KM3Aux -export meaningoflife +using Printf + +export filepath + + +function getcalibpath() + p = get(ENV, "KM3AUX_CALIB_PATH", "") + if p == "" + error( + "The environment variable 'KM3AUX_CALIB_PATH' is not set. " * + "It should point to a clone of the " * + "https://git.km3net.de/auxiliary_data/calibration repository." + ) + end + if !ispath(p) + error( + "The KM3AUX_CALIB_PATH environment variable points to a " * + "non-existent directory." + ) + end + if !isdir(p) + error( + "The KM3AUX_CALIB_PATH environment variable points to an " * + "invalid directory." + ) + end + p +end + + +""" + filepath(detid::Int, runid::Int, type::AbstractString) + +Returns the filepath for an auxiliary file based on the detector ID, run and type. +The environment variable `KM3AUX_CALIB_PATH` is used as base directory and the latest +version of the file is picked. The auxiliary archive is using LFS and the file will +be pulled automatically. + +The following version scheme is used in the archive: + + ./XXXXXXXX/<type>/_<version>/YYYYYYYY/ZZZZZZZZ/<counter>.<extension> + +""" +function filepath(detid::Int, runid::Int, type::AbstractString) + calibpath = getcalibpath() + detpath = joinpath(calibpath, zeropad8(detid)) + !isdir(detpath) && error("No data found for detector with ID '$detid'.") + + typepath = joinpath(detpath, type) + if !isdir(typepath) + available_types = join(readdir(detpath), ", ", " and ") + error("Unknown type '$type'. Available types: $(available_types).") + end + + versions = readdir(typepath) |> versionsort + length(versions) == 0 && error("No data of type '$type' found for detector with ID '$detid'.") + + v = versions[end] + versionpath = joinpath(typepath, v) + + runs = sort([parse(Int, run) for run in readdir(versionpath)]) + run_begin_idx = searchsortedlast(runs, runid) + run_begin_idx == 0 && error("No data of type '$type' found for detector with ID '$detid' and run '$runid'.") + run_begin = runs[run_begin_idx] + + runpath = joinpath(versionpath, zeropad8(run_begin)) + run_end = readdir(runpath) + length(run_end) == 0 && error("Incomplete data of type '$type' for detector with ID '$detid' and run '$runid'.") + + auxpath = joinpath(runpath, run_end[end]) # run_end should only have one element + fnames = readdir(auxpath) |> versionsort + length(fnames) == 0 && error("Missing data of type '$type' for detector with ID '$detid' and run '$runid'.") + + fpath = joinpath(auxpath, fnames[end]) + + !needs_lfs_pull(fpath) && return fpath + + # pulling from LFS + lfspath = relpath(fpath, calibpath) + # wait=false to suppress output of the Git command + process = run(Cmd(`git lfs pull --exclude= --include $lfspath`, dir=calibpath, ignorestatus=true), wait=false) + # we need to wait for it... + wait(process) + if process.exitcode != 0 + command = join(process.cmd.exec, " ") + error( + "Git LFS command failed with non-zero exit code ($(process.exitcode)).\n" * + " Command: $command" + ) + end + + fpath +end """ - meaningoflife() + needs_lfs_pull(fpath::AbstractString) + +Checks if a given filepath is an LFS object yet to be pulled. +""" +function needs_lfs_pull(fpath::AbstractString) + s = String(read(fpath, 39)) + s == "version https://git-lfs.github.com/spec" +end + + +""" + zeropad8(x::Integer) + +Creates an 8 characters long zeropadded string for a given number. + +# Examples + +``` +julia> KM3Aux.zeropad8(42) +"00000042" +""" +zeropad8(x::Integer) = @sprintf "%08d" x + + +""" + versionsort(v::Vector{T}) where T<:AbstractString + +Sorts a collection of strings based on version identifiers. +""" +function versionsort(v::Vector{T}) where T<:AbstractString + sort(v; by=expandversion) +end + + +""" + expandversion(s::AbstractString) + +Expands the version components into tuples, useful for sorting. + +# Examples + +``` +julia> KM3Aux.expandversion("_H_1.12.3") +("H", 1, 12, 3) -Determines the meaning of life without floating point precision. +julia> KM3Aux.expandversion("42.datx") +42 +``` """ -function meaningoflife() - a = 21 - b = 2 - return a * b +function expandversion(s::AbstractString) + m = match(r"_(\w)_(\d+)\.(\d+)\.(\d+)", s) + if !isnothing(m) + return(m[1], parse(Int, m[2]), parse(Int, m[3]), parse(Int, m[4])) + else + m = match(r"(\d+)\.?+", s) + if !isnothing(m) + return parse(Int, m[1]) + end + end + return s end end diff --git a/test/misc.jl b/test/misc.jl deleted file mode 100644 index 3f014095225b0e3cf3df2125f274f5c7ec66543c..0000000000000000000000000000000000000000 --- a/test/misc.jl +++ /dev/null @@ -1,7 +0,0 @@ -using KM3Aux -using Test - - -@testset "meaningoflife()" begin - @test 42 == meaningoflife() -end diff --git a/test/runtests.jl b/test/runtests.jl index 9be2b8bc74865688fd18c1bd5c77995eea74f0e0..04bd987d911116a221b80d625df5edbfc384e895 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,3 +1,9 @@ using Test +using KM3Aux -include("misc.jl") +@testset "getfilepath" begin + calibpath = joinpath(@__DIR__, "calibration") + ENV["KM3AUX_CALIB_PATH"] = calibpath + + @test filepath(160, 19466, "pmt") == joinpath(calibpath, "00000160", "pmt", "_H_10.1.3", "00019465", "00019532", "10.txt") +end