diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 124a32cbc0afe1aa03a178eabe75c7e2c98e4a71..da72ebd61c8344f67cd99889d66a162da6655f56 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,13 +29,17 @@ build: paths: - Deliverable_4-8.md - Deliverable_4-9.md + - statuspage.md expire_in: 3 weeks # the 'pages' job will deploy and build your site to the 'public' path pages: stage: deploy + dependencies: + - build script: - gitbook build . public # build to public path + - mv statuspage.md public artifacts: paths: - public diff --git a/SUMMARY.md b/SUMMARY.md index e345e71261ceda2ed162290bd0bc49b281bafddd..0189b64e2a33ec841a2b41cde048bef5346aaf66 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -39,4 +39,5 @@ * Data publication ## [References](pages/References.md) +## [Deliverable status](statuspage.md) diff --git a/filelist_D4-8.txt b/filelist_D4-8.txt index c00300d2b3f773a0ce373d0bfcf7a0687b276874..8d8baa19f8d973a9deae69f5a198fe3228bd238e 100644 --- a/filelist_D4-8.txt +++ b/filelist_D4-8.txt @@ -16,8 +16,7 @@ pages/VOserver.md pages/KM3NeTserver.md pages/Git.md pages/Docker.md -pages/Repositories_VOserver.md -pages/Repositories_Zenodo.md +pages/Repositories.md pages/Courses.md pages/OSP.md pages/Python.md diff --git a/make_documents.py b/make_documents.py index 612823676adc160d470718685fafde8d8d1cd1a0..4fa04ab6bda3bca4c2b01f547ae357eae1069d09 100644 --- a/make_documents.py +++ b/make_documents.py @@ -1,32 +1,55 @@ #!/usr/bin/python def fulltext_from_filelist(filelistname, onlystatus = ""): - fulltext = "" + fulltext = "Summary for "+filenamelist+"\n\n" + fulltext += "[[_TOC_]]" + overview = {"filename": [], "author": [], "status": []} with open(filelistname, "r") as f: infilename = f.readline().rstrip("\n") while infilename: with open(infilename, "r") as ftext: addtext = ftext.read() - #info = yaml.load(addtext[addtext.find("---")+3:addtext.find("---", 5)]) + infotext = addtext[addtext.find("---")+3:addtext.find("---", 5)].split("\n") + status, author = "", "" + for line in infotext: + if line.find("status:")> -1: + status = line[9:len(line)] + if line.find("Author")> -1: + author = line[9:len(line)] takeit = True - '''if onlystatus: + if onlystatus: takeit = False - if "status" in info: - if info["status"] == onlystatus: - takeit = True''' + if status: + if status == onlystatus: + takeit = True if takeit: fulltext += addtext[addtext.find("---", 5)+3:len(addtext)] + "\n" + overview["filename"].append(infilename) + overview["author"].append(author) + overview["status"].append(status) infilename = f.readline().rstrip("\n") - return fulltext + + return fulltext, overview -deliverable_description = fulltext_from_filelist("filelist_D4-8.txt") +deliverable_description, status1 = fulltext_from_filelist("filelist_D4-8.txt") with open("Deliverable_4-8.md", "w") as f: f.write(deliverable_description) -deliverable_manuals = fulltext_from_filelist("filelist_D4-9.txt") +deliverable_manuals, status2 = fulltext_from_filelist("filelist_D4-9.txt") with open("Deliverable_4-9.md", "w") as f: f.write(deliverable_manuals) +with open("statuspage.md", "w") as f: + status = "## Current page status\n\n" + status += "| Filename | Author | Status |\n" + status += "| -------- | ------ | ------ |\n" + for stats in (status1, status2) + for i in range(len(stats["filename"])): + status += "| " + stats["filename"] \ + + " | " + stats["author"] \ + + " | " + stats["status"] + " |\n" + f.write(status) + diff --git a/pages/Courses.md b/pages/Courses.md index 581d2e0924e22a4461a92b2b17bc0da4937313e2..e9f2ed3361915262caf14bb20dfc982cbe60c548 100644 --- a/pages/Courses.md +++ b/pages/Courses.md @@ -5,6 +5,7 @@ Topics: - summary on education center (from D4.10) - summary on webinar - How to use the webinar materials +status: unedited --- Please add links and summary to open courses in edu portal diff --git a/pages/Dataformats.md b/pages/Dataformats.md index ab7113ee6358090f29331828ee8d0b09cb439ce7..f8c25fbf459e3f0e2592b63db086d64254a71700 100644 --- a/pages/Dataformats.md +++ b/pages/Dataformats.md @@ -4,35 +4,13 @@ Author: Jutta status: dump --- -# Open data formats +## Open data products -## Neutrino sets in the VO +### Neutrino sets in the VO -Tabulated high-level neutrino event data can be provided through the VO -registry, utilizing access protocols like the Table Access Protocol -(TAP) and query languages like the Astronomical Data Query Language -(ADQL). To query these data sets related to astronomical sources, the -Simple Cone Search (SCS) protocol allows to pick specific events -according to particle source direction, using Unified Content -Descriptors (UCDs) to identify the relevant table columns. The -underlying data format is the VOTable which allows for metadata -annotation of data columns. As the DaCHS software provides input -capabilities for various formats like FITS\footnote{Flexible Image - Transport System, \url{https://fits.gsfc.nasa.gov/}.} or text-based -tables on the server side, a common KM3NeT open event table format -can be chosen quite independently and the interface adapted such that high-level neutrino data -sets can be both offered through the VO and alternative access -protocols, as long as the required metadata description is handled -adequately. -VO standards are at the current stage not fully adapted -to the inclusion of neutrino data and require development of metadata -standards for easy interpretability of the data, a matter which is targeted within the ESCAPE project. Open questions in this regard are the linkage of observation probabilities to a given event selection, the inclusion of ``non-observation'' in a given field of view -and within a given time as relevant scientific information to retrieve -from the services, and the introduction of a dedicated vocabulary for the -description of neutrino data. This vocabulary will need to be developed within KM3NeT as a matter of internal standardization, however, the process will draw guidance from the VO expertise and framework. -## Multimessenger alerts +### Multimessenger alerts Single or stacked neutrino events of high astrophysical signal probability will be selected in KM3NeT to trigger an alert to other observatories indicating a possible target for multimessenger observations \cite{MM}. The diff --git a/pages/Datamodels.md b/pages/Datamodels.md index f4bf99837f0af459bf8333589849d5ca51bcde37..743080fc60be9e0bb249b7255e363f1ebad03871 100644 --- a/pages/Datamodels.md +++ b/pages/Datamodels.md @@ -4,6 +4,7 @@ Author: Jutta Topics: - data models - configurations for software +status: dump --- * Started collection of metadata definitions (from publication level), based on standards of e.g. W3C, IVOA, DataCite etc. diff --git a/pages/Dataserver_api.md b/pages/Dataserver_api.md index 6835cb8e67737049aa400278914e3de4f7a15a9f..e1c06fa0192e9baf4e6da56398b12a4ffca224a4 100644 --- a/pages/Dataserver_api.md +++ b/pages/Dataserver_api.md @@ -1,4 +1,5 @@ --- Title: KM3NeT server REST API Author: Jutta +status: unedited --- diff --git a/pages/Detector.md b/pages/Detector.md index f3a577b68e70a735454f9f6e0cb38e83d0d86238..9ba6288632816bb1a1a685661c16b9a334a0f9b5 100644 --- a/pages/Detector.md +++ b/pages/Detector.md @@ -1,7 +1,7 @@ --- Title: Detector and Data Taking Author: Jannik - +status: draft Comment: Mainly based on LoI --- diff --git a/pages/Docker.md b/pages/Docker.md index d1a6602261d67964fe434754ad0b2826109ab802..451ce8b3f61a49aaa95f8bed40231b76f7a08f68 100644 --- a/pages/Docker.md +++ b/pages/Docker.md @@ -5,4 +5,5 @@ Title: - short description - software interoperability - archiving of software +status: unedited --- diff --git a/pages/ESCAPE.md b/pages/ESCAPE.md index cab9dadc0908309f28b29ed117e211a7dd141512..65e3f1f6d5afba7c5eb5b19be6b3f14af474b1b8 100644 --- a/pages/ESCAPE.md +++ b/pages/ESCAPE.md @@ -1,4 +1,11 @@ --- Title: ESAP & ESCAPE Author: Jutta +status: dump --- + +## Further integration with the Virtual Observatory + +VO standards are at the current stage not fully adapted +to the inclusion of neutrino data and require development of metadata +standards for easy interpretability of the data, a matter which is targeted within the ESCAPE project. Open questions in this regard are the linkage of observation probabilities to a given event selection, the inclusion of ``non-observation'' in a given field of view and within a given time as relevant scientific information to retrieve from the services, and the introduction of a dedicated vocabulary for the description of neutrino data. This vocabulary will need to be developed within KM3NeT as a matter of internal standardization, however, the process will draw guidance from the VO expertise and framework. diff --git a/pages/FAIR.md b/pages/FAIR.md index 7c76249cb055f36168723e3db8037c0f621294b0..32f3e3fdde7f003e6e48e2e5006e5653c543c2e3 100644 --- a/pages/FAIR.md +++ b/pages/FAIR.md @@ -4,6 +4,7 @@ Author: Jutta Topics: - Policy basics - dedication to open science +status: dump --- ## Publishing FAIR data diff --git a/pages/Git.md b/pages/Git.md index 31aeb63fb8223c319885c934f478d639e0f5d344..6d8b5a215a44ac40998a0bd3def1cb9d64054771 100644 --- a/pages/Git.md +++ b/pages/Git.md @@ -5,6 +5,7 @@ Title: - use for software, plots and analysis - integration to github - Documentation on Git & automatic software docs +status: dump --- KM3NeT uses GitLab as the main platform to develop and discuss software, diff --git a/pages/Infrastructure.md b/pages/Infrastructure.md index 95eaf50da97be9083267aeecbe0d43a1892df416..273b9fba6ab7704944a83c07644a1135f51aed22 100644 --- a/pages/Infrastructure.md +++ b/pages/Infrastructure.md @@ -3,5 +3,12 @@ Title: Architecture overview Author: Jutta Topics: - servers, repositories, webpages +status: dump --- +## Platforms for Open Science + +However, considering the role of neutrino physics just at the edge of use for astronomical studies, KM3NeT data +integration also meets some limitations considering the scientific +usability of the provided data sets. In addition to that, publication of data through VO standards is clearly limited to astronomy-related data in a celestial reference frame. + diff --git a/pages/KM3NeTserver.md b/pages/KM3NeTserver.md index de731f65a6a6387545cc6a26d0ae966aa46e5367..fb6699a58ee098131f3325c0d3a620840b7c628d 100644 --- a/pages/KM3NeTserver.md +++ b/pages/KM3NeTserver.md @@ -1,6 +1,7 @@ --- Title: KM3NeT server Author: Jutta +status: dump --- * For all data not publishable through the IVOA, serving as interface and/or server to the data diff --git a/pages/Multimessenger.md b/pages/Multimessenger.md index f757ed7c644e82743e55095414063d3c447a043d..ac525ba010f101a45c24e3490a43ba20dbe2af48 100644 --- a/pages/Multimessenger.md +++ b/pages/Multimessenger.md @@ -4,4 +4,5 @@ Author: Feifei Topics: - alert types - short description of the system +status: unedited --- diff --git a/pages/OSP.md b/pages/OSP.md index fbab904771fbaf5f89559570fecedb6b82efc07c..41088eddba68bb2c8f2be67f25b06921e65b046d 100644 --- a/pages/OSP.md +++ b/pages/OSP.md @@ -4,6 +4,7 @@ Author: Jutta Topics: - installation - interface through software +status: dump --- * Git project serving webpage (Gitbook) diff --git a/pages/Procedures.md b/pages/Procedures.md index db4ebd5f8c8f8e147dc4b0a62c3b30a6d0a79d40..8ca343c9822e9b7c2ec8b94c63ceba23ec49f4e5 100644 --- a/pages/Procedures.md +++ b/pages/Procedures.md @@ -4,6 +4,7 @@ Author: Kay, Jutts Topics: - data/software releases - publication procedures +status: dump --- ## Establishing the Open Science Committee diff --git a/pages/Processing.md b/pages/Processing.md index c01bbc20ca60dc396df2293eaaaead6075f4ef3b..15ccbc1fdd74fb4ac8fc112e165e435690d39da3 100644 --- a/pages/Processing.md +++ b/pages/Processing.md @@ -1,6 +1,7 @@ --- Title: Processing overview Author: Jutta +status: dump --- Data processing therefore follows a tier-based approach \cite{km3comp}, where initial diff --git a/pages/Python.md b/pages/Python.md index 263d86b0910b297da4c311865be3b29d2aefbb8a..ebc6459562e134835ea231fb61540dd34ca1d33d 100644 --- a/pages/Python.md +++ b/pages/Python.md @@ -4,6 +4,7 @@ Authors: Tamas, Jutta Topics: - installation - interface through software +status: dump --- diff --git a/pages/Quality.md b/pages/Quality.md index 0ecb0fba4246d6d0442241fa39d2b2530b15dfcb..acd20bdebd9d660d22d911280615c9170032614a 100644 --- a/pages/Quality.md +++ b/pages/Quality.md @@ -4,4 +4,5 @@ Author: Rodri Topics: - quality checks on event data - quality indicators +status: unedited --- diff --git a/pages/Repositories_Zenodo.md b/pages/Repositories.md similarity index 97% rename from pages/Repositories_Zenodo.md rename to pages/Repositories.md index 1a2a2ff7bb512c862df1bb55e3a76c6395ca3739..e05549a1f061e38208d30ed12954ae7e14dd5e2a 100644 --- a/pages/Repositories_Zenodo.md +++ b/pages/Repositories.md @@ -1,6 +1,7 @@ --- Title: Integrating to Zenodo Author: Jutta +status: dump --- * For findability data needs to be citable and registered within a large repository (e.g. DataCite) diff --git a/pages/Repositories_VOserver.md b/pages/Repositories_VOserver.md deleted file mode 100644 index c4e9839bdbde2ba2573de51351fb08d32bf649f9..0000000000000000000000000000000000000000 --- a/pages/Repositories_VOserver.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -Title: VO repository -Author: Jutta ---- diff --git a/pages/ScienceTargets.md b/pages/ScienceTargets.md index a3312255c0f988713676416c43cf421e2f020b91..5e12eaa740a3bc8424127ce8fd0d10989b2328c0 100644 --- a/pages/ScienceTargets.md +++ b/pages/ScienceTargets.md @@ -1,6 +1,7 @@ --- Title: Scientific targets Author: Jannik +status: draft --- # Scientific Targets diff --git a/pages/SeaScience.md b/pages/SeaScience.md index 08e416d2cf1dfd5a8599d8218eb1c0295c41719e..2ad2052f63ae67b553dad44b42c94a3a0613d96e 100644 --- a/pages/SeaScience.md +++ b/pages/SeaScience.md @@ -4,7 +4,7 @@ Author: Jannik Topics: - options for sea science - types of environmental data -Status: draft +status: draft --- ## Environmental data diff --git a/pages/Simulation.md b/pages/Simulation.md index 1b6d3163474cca519bafda24c2bc5d8b5ef66b91..2a1c62bd248d88136676d18677b9fec253abfedc 100644 --- a/pages/Simulation.md +++ b/pages/Simulation.md @@ -1,6 +1,7 @@ --- Title: Event simulation Author: Jutta, Tamas +status: dump --- In order to evaluate and test physical models of neutrino interactions and productions, a huge variety of simulations need to be performed and compared to the data taken with the KM3NeT detectors. diff --git a/pages/Softwaredevelopment.md b/pages/Softwaredevelopment.md index b6548514daebf871e619784d83fe2e6c7ee25d0c..701b7bd5c6eff28aeed004936f151562cac6a544 100644 --- a/pages/Softwaredevelopment.md +++ b/pages/Softwaredevelopment.md @@ -4,4 +4,5 @@ Author: Tamas Topics: - using the python template - software development recommendations +status: unedited --- diff --git a/pages/Usecase_ANTARES.md b/pages/Usecase_ANTARES.md index 4eef6d42a71a40db4ba02fdf6f9bb2b6df99c995..3bb87eadf9e5a97f8752cdc5e7520dadb775c5b2 100644 --- a/pages/Usecase_ANTARES.md +++ b/pages/Usecase_ANTARES.md @@ -5,4 +5,5 @@ Topics: - short description analysis - data description - linking to notebooks & background +status: unedited --- diff --git a/pages/Usecase_Environmental.md b/pages/Usecase_Environmental.md index 3520f9ae08202da0e36d3348146b5ed212f15976..29e3c0a9fb90fbd8a06f6a355245840646fe29b1 100644 --- a/pages/Usecase_Environmental.md +++ b/pages/Usecase_Environmental.md @@ -4,6 +4,7 @@ Topics: - short description use - data description - interfaces +status: dump --- * Acoustic data will be provided through separate REST-API diff --git a/pages/Usecase_KM3NeT.md b/pages/Usecase_KM3NeT.md index 4bdbcc90cb4233127e3cd6fd8d8001bd099c2009..cf9ddbf9a56e412bdac256baa3d789faa0abe38d 100644 --- a/pages/Usecase_KM3NeT.md +++ b/pages/Usecase_KM3NeT.md @@ -5,6 +5,7 @@ Topics: - short description analysis - data description - linking to notebooks & background +status: unedited --- diff --git a/pages/Using_alerts.md b/pages/Using_alerts.md index a3230010c1150976408cb97acd1d7ba09267402e..6b8eb79021bf5e63af796062217896c0c1d44985 100644 --- a/pages/Using_alerts.md +++ b/pages/Using_alerts.md @@ -4,8 +4,11 @@ Author: Feifei, Jutta Topics: - how to get the alerts? - Description of VOEvent +status: unedited +Comment: | + To create such a page from the notebook, + export the jupyter notebook as markdown + and add to this text file and store the notebook + in the notebooks/ folder --- -To create such a page from the notebook, -* export the jupyter notebook as markdown and add to this text file -* store the notebook in the notebooks/ folder diff --git a/pages/Using_km3events.md b/pages/Using_km3events.md index 8bc180522eb7aede51ec42ea8ba3f427d69690ec..9d2b3d35e86edee18d6dbafe5f3592523bd922cc 100644 --- a/pages/Using_km3events.md +++ b/pages/Using_km3events.md @@ -4,8 +4,11 @@ Author: Steffen, Jutta Topics: - describe download - notebook +status: unedited +Comment: | + To create such a page from the notebook, + export the jupyter notebook as markdown + and add to this text file and store the notebook + in the notebooks/ folder --- -To create such a page from the notebook, -* export the jupyter notebook as markdown and add to this text file -* store the notebook in the notebooks/ folder diff --git a/pages/Using_plotdata.md b/pages/Using_plotdata.md index 2f824ab16a0b34f35c4a23bc2235de18496dba07..e17e67ac06e59da5ad2fead3b4e8228e98589861 100644 --- a/pages/Using_plotdata.md +++ b/pages/Using_plotdata.md @@ -4,8 +4,12 @@ Author: Rebecca, Jutta Topics: - ANTARES notebook example - openkm3 example +status: unedited +Comment: | + To create such a page from the notebook, + export the jupyter notebook as markdown + and add to this text file and store the notebook + in the notebooks/ folder --- -To create such a page from the notebook, -* export the jupyter notebook as markdown and add to this text file -* store the notebook in the notebooks/ folder + diff --git a/pages/Using_templates.md b/pages/Using_templates.md index caa38cab100afa3e60f6bd4634787a17553582ee..fc48ee41ee5f0da7aea079b3fce41e17c1856619 100644 --- a/pages/Using_templates.md +++ b/pages/Using_templates.md @@ -6,4 +6,5 @@ Topics: - guidelines - git analysis template description - guidelines +status: unedited --- diff --git a/pages/Using_vo.md b/pages/Using_vo.md index ae8cea6e72a76dfc20093e997375ca8180ecbcda..035e483536af5b7b6280c21707117cd6527de14b 100644 --- a/pages/Using_vo.md +++ b/pages/Using_vo.md @@ -5,7 +5,7 @@ Topics: - summary from open course - https://edu.km3net.de/course/accessing-the-km3net-open-data/ -Status: draft +status: dump --- # Accessing data through the VO server diff --git a/pages/VOserver.md b/pages/VOserver.md index 5cf2df8f3ed06a5c6a688ab9832cb29c9ff23a02..498a34493b7650fa41c1bdbd929b748a963495aa 100644 --- a/pages/VOserver.md +++ b/pages/VOserver.md @@ -1,23 +1,21 @@ --- Title: VOserver Author: Jutta +status: unedited --- -## Approach to the Virtual Observatory standard +## The Virtual Observatory server -The Virtual Observatory (VO) standards\cite{VO} -serve to create an interface between astronomy-related data resources -from astrophysics experiments which act as data providers. The focus is on the scientific end user to easily interface from their personal -computer with the provided data sets. The KM3NeT collaboration is a data -provider to the VO and operates a data server\footnote{at \url{http://vo.km3net.de/}} running the DaCHS software\cite{dachs}. +## Publication + +The Virtual Observatory (VO) standards serve to create an interface between astronomy-related data resources from astrophysics experiments which act as data providers. The focus is on the scientific end user to easily interface from their personal +computer with the provided data sets. The KM3NeT collaboration is a data provider to the VO and operates a data server at http://vo.km3net.de/ running the (DaCHS software)[http://docs.g-vo.org/DaCHS/]. The well-developed data sharing regime of the VO serves well as a guideline for the implementation of astrophysical data sharing in the -KM3NeT collaboration. However, considering the role of neutrino physics -just at the edge of use for astronomical studies, KM3NeT data -integration also meets some limitations considering the scientific -usability of the provided data sets. In addition to that, publication of -data through VO standards is clearly limited to astronomy-related data -in a celestial reference frame. +KM3NeT collaboration. + +## Data access +Tabulated high-level neutrino event data can be provided through the VO registry, utilizing access protocols like the Table Access Protocol (TAP) and query languages like the Astronomical Data Query Language (ADQL). To query these data sets related to astronomical sources, the Simple Cone Search (SCS) protocol allows to pick specific events according to particle source direction. * Running server with DaCHS software * Registered as data provider to the VO (ivo://km3net.org) diff --git a/pages/Workflows.md b/pages/Workflows.md index 8fdd493f99ef2d50cbf2ab370c8ff86a3bdd9720..131c5c02e3ce81dbd216b833ec26325820e58781 100644 --- a/pages/Workflows.md +++ b/pages/Workflows.md @@ -1,6 +1,7 @@ --- Title: Workflow management Author: Jutta, Tamas +status: dump --- The simulation and raw data processing chains consist of workflows that are