From 06f807075ae5e88f59179ca96b945d066b000853 Mon Sep 17 00:00:00 2001
From: Jutta Schnabel <jschnabel@km3net.de>
Date: Fri, 9 Oct 2020 23:15:27 +0200
Subject: [PATCH] adding status
---
.gitlab-ci.yml | 4 ++
SUMMARY.md | 1 +
filelist_D4-8.txt | 3 +-
make_documents.py | 41 +++++++++++++++----
pages/Courses.md | 1 +
pages/Dataformats.md | 28 ++-----------
pages/Datamodels.md | 1 +
pages/Dataserver_api.md | 1 +
pages/Detector.md | 2 +-
pages/Docker.md | 1 +
pages/ESCAPE.md | 7 ++++
pages/FAIR.md | 1 +
pages/Git.md | 1 +
pages/Infrastructure.md | 7 ++++
pages/KM3NeTserver.md | 1 +
pages/Multimessenger.md | 1 +
pages/OSP.md | 1 +
pages/Procedures.md | 1 +
pages/Processing.md | 1 +
pages/Python.md | 1 +
pages/Quality.md | 1 +
...Repositories_Zenodo.md => Repositories.md} | 1 +
pages/Repositories_VOserver.md | 4 --
pages/ScienceTargets.md | 1 +
pages/SeaScience.md | 2 +-
pages/Simulation.md | 1 +
pages/Softwaredevelopment.md | 1 +
pages/Usecase_ANTARES.md | 1 +
pages/Usecase_Environmental.md | 1 +
pages/Usecase_KM3NeT.md | 1 +
pages/Using_alerts.md | 9 ++--
pages/Using_km3events.md | 9 ++--
pages/Using_plotdata.md | 10 +++--
pages/Using_templates.md | 1 +
pages/Using_vo.md | 2 +-
pages/VOserver.md | 22 +++++-----
pages/Workflows.md | 1 +
37 files changed, 109 insertions(+), 64 deletions(-)
rename pages/{Repositories_Zenodo.md => Repositories.md} (97%)
delete mode 100644 pages/Repositories_VOserver.md
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 124a32c..da72ebd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,13 +29,17 @@ build:
paths:
- Deliverable_4-8.md
- Deliverable_4-9.md
+ - statuspage.md
expire_in: 3 weeks
# the 'pages' job will deploy and build your site to the 'public' path
pages:
stage: deploy
+ dependencies:
+ - build
script:
- gitbook build . public # build to public path
+ - mv statuspage.md public
artifacts:
paths:
- public
diff --git a/SUMMARY.md b/SUMMARY.md
index e345e71..0189b64 100644
--- a/SUMMARY.md
+++ b/SUMMARY.md
@@ -39,4 +39,5 @@
* Data publication
## [References](pages/References.md)
+## [Deliverable status](statuspage.md)
diff --git a/filelist_D4-8.txt b/filelist_D4-8.txt
index c00300d..8d8baa1 100644
--- a/filelist_D4-8.txt
+++ b/filelist_D4-8.txt
@@ -16,8 +16,7 @@ pages/VOserver.md
pages/KM3NeTserver.md
pages/Git.md
pages/Docker.md
-pages/Repositories_VOserver.md
-pages/Repositories_Zenodo.md
+pages/Repositories.md
pages/Courses.md
pages/OSP.md
pages/Python.md
diff --git a/make_documents.py b/make_documents.py
index 6128236..4fa04ab 100644
--- a/make_documents.py
+++ b/make_documents.py
@@ -1,32 +1,55 @@
#!/usr/bin/python
def fulltext_from_filelist(filelistname, onlystatus = ""):
- fulltext = ""
+ fulltext = "Summary for "+filenamelist+"\n\n"
+ fulltext += "[[_TOC_]]"
+ overview = {"filename": [], "author": [], "status": []}
with open(filelistname, "r") as f:
infilename = f.readline().rstrip("\n")
while infilename:
with open(infilename, "r") as ftext:
addtext = ftext.read()
- #info = yaml.load(addtext[addtext.find("---")+3:addtext.find("---", 5)])
+ infotext = addtext[addtext.find("---")+3:addtext.find("---", 5)].split("\n")
+ status, author = "", ""
+ for line in infotext:
+ if line.find("status:")> -1:
+ status = line[9:len(line)]
+ if line.find("Author")> -1:
+ author = line[9:len(line)]
takeit = True
- '''if onlystatus:
+ if onlystatus:
takeit = False
- if "status" in info:
- if info["status"] == onlystatus:
- takeit = True'''
+ if status:
+ if status == onlystatus:
+ takeit = True
if takeit:
fulltext += addtext[addtext.find("---", 5)+3:len(addtext)] + "\n"
+ overview["filename"].append(infilename)
+ overview["author"].append(author)
+ overview["status"].append(status)
infilename = f.readline().rstrip("\n")
- return fulltext
+
+ return fulltext, overview
-deliverable_description = fulltext_from_filelist("filelist_D4-8.txt")
+deliverable_description, status1 = fulltext_from_filelist("filelist_D4-8.txt")
with open("Deliverable_4-8.md", "w") as f:
f.write(deliverable_description)
-deliverable_manuals = fulltext_from_filelist("filelist_D4-9.txt")
+deliverable_manuals, status2 = fulltext_from_filelist("filelist_D4-9.txt")
with open("Deliverable_4-9.md", "w") as f:
f.write(deliverable_manuals)
+with open("statuspage.md", "w") as f:
+ status = "## Current page status\n\n"
+ status += "| Filename | Author | Status |\n"
+ status += "| -------- | ------ | ------ |\n"
+ for stats in (status1, status2)
+ for i in range(len(stats["filename"])):
+ status += "| " + stats["filename"] \
+ + " | " + stats["author"] \
+ + " | " + stats["status"] + " |\n"
+ f.write(status)
+
diff --git a/pages/Courses.md b/pages/Courses.md
index 581d2e0..e9f2ed3 100644
--- a/pages/Courses.md
+++ b/pages/Courses.md
@@ -5,6 +5,7 @@ Topics:
- summary on education center (from D4.10)
- summary on webinar
- How to use the webinar materials
+status: unedited
---
Please add links and summary to open courses in edu portal
diff --git a/pages/Dataformats.md b/pages/Dataformats.md
index ab7113e..f8c25fb 100644
--- a/pages/Dataformats.md
+++ b/pages/Dataformats.md
@@ -4,35 +4,13 @@ Author: Jutta
status: dump
---
-# Open data formats
+## Open data products
-## Neutrino sets in the VO
+### Neutrino sets in the VO
-Tabulated high-level neutrino event data can be provided through the VO
-registry, utilizing access protocols like the Table Access Protocol
-(TAP) and query languages like the Astronomical Data Query Language
-(ADQL). To query these data sets related to astronomical sources, the
-Simple Cone Search (SCS) protocol allows to pick specific events
-according to particle source direction, using Unified Content
-Descriptors (UCDs) to identify the relevant table columns. The
-underlying data format is the VOTable which allows for metadata
-annotation of data columns. As the DaCHS software provides input
-capabilities for various formats like FITS\footnote{Flexible Image
- Transport System, \url{https://fits.gsfc.nasa.gov/}.} or text-based
-tables on the server side, a common KM3NeT open event table format
-can be chosen quite independently and the interface adapted such that high-level neutrino data
-sets can be both offered through the VO and alternative access
-protocols, as long as the required metadata description is handled
-adequately.
-VO standards are at the current stage not fully adapted
-to the inclusion of neutrino data and require development of metadata
-standards for easy interpretability of the data, a matter which is targeted within the ESCAPE project. Open questions in this regard are the linkage of observation probabilities to a given event selection, the inclusion of ``non-observation'' in a given field of view
-and within a given time as relevant scientific information to retrieve
-from the services, and the introduction of a dedicated vocabulary for the
-description of neutrino data. This vocabulary will need to be developed within KM3NeT as a matter of internal standardization, however, the process will draw guidance from the VO expertise and framework.
-## Multimessenger alerts
+### Multimessenger alerts
Single or stacked neutrino events of high astrophysical signal probability will be selected in KM3NeT to trigger an alert to other observatories indicating a
possible target for multimessenger observations \cite{MM}. The
diff --git a/pages/Datamodels.md b/pages/Datamodels.md
index f4bf998..743080f 100644
--- a/pages/Datamodels.md
+++ b/pages/Datamodels.md
@@ -4,6 +4,7 @@ Author: Jutta
Topics:
- data models
- configurations for software
+status: dump
---
* Started collection of metadata definitions (from publication level), based on standards of e.g. W3C, IVOA, DataCite etc.
diff --git a/pages/Dataserver_api.md b/pages/Dataserver_api.md
index 6835cb8..e1c06fa 100644
--- a/pages/Dataserver_api.md
+++ b/pages/Dataserver_api.md
@@ -1,4 +1,5 @@
---
Title: KM3NeT server REST API
Author: Jutta
+status: unedited
---
diff --git a/pages/Detector.md b/pages/Detector.md
index f3a577b..9ba6288 100644
--- a/pages/Detector.md
+++ b/pages/Detector.md
@@ -1,7 +1,7 @@
---
Title: Detector and Data Taking
Author: Jannik
-
+status: draft
Comment: Mainly based on LoI
---
diff --git a/pages/Docker.md b/pages/Docker.md
index d1a6602..451ce8b 100644
--- a/pages/Docker.md
+++ b/pages/Docker.md
@@ -5,4 +5,5 @@ Title:
- short description
- software interoperability
- archiving of software
+status: unedited
---
diff --git a/pages/ESCAPE.md b/pages/ESCAPE.md
index cab9dad..65e3f1f 100644
--- a/pages/ESCAPE.md
+++ b/pages/ESCAPE.md
@@ -1,4 +1,11 @@
---
Title: ESAP & ESCAPE
Author: Jutta
+status: dump
---
+
+## Further integration with the Virtual Observatory
+
+VO standards are at the current stage not fully adapted
+to the inclusion of neutrino data and require development of metadata
+standards for easy interpretability of the data, a matter which is targeted within the ESCAPE project. Open questions in this regard are the linkage of observation probabilities to a given event selection, the inclusion of ``non-observation'' in a given field of view and within a given time as relevant scientific information to retrieve from the services, and the introduction of a dedicated vocabulary for the description of neutrino data. This vocabulary will need to be developed within KM3NeT as a matter of internal standardization, however, the process will draw guidance from the VO expertise and framework.
diff --git a/pages/FAIR.md b/pages/FAIR.md
index 7c76249..32f3e3f 100644
--- a/pages/FAIR.md
+++ b/pages/FAIR.md
@@ -4,6 +4,7 @@ Author: Jutta
Topics:
- Policy basics
- dedication to open science
+status: dump
---
## Publishing FAIR data
diff --git a/pages/Git.md b/pages/Git.md
index 31aeb63..6d8b5a2 100644
--- a/pages/Git.md
+++ b/pages/Git.md
@@ -5,6 +5,7 @@ Title:
- use for software, plots and analysis
- integration to github
- Documentation on Git & automatic software docs
+status: dump
---
KM3NeT uses GitLab as the main platform to develop and discuss software,
diff --git a/pages/Infrastructure.md b/pages/Infrastructure.md
index 95eaf50..273b9fb 100644
--- a/pages/Infrastructure.md
+++ b/pages/Infrastructure.md
@@ -3,5 +3,12 @@ Title: Architecture overview
Author: Jutta
Topics:
- servers, repositories, webpages
+status: dump
---
+## Platforms for Open Science
+
+However, considering the role of neutrino physics just at the edge of use for astronomical studies, KM3NeT data
+integration also meets some limitations considering the scientific
+usability of the provided data sets. In addition to that, publication of data through VO standards is clearly limited to astronomy-related data in a celestial reference frame.
+
diff --git a/pages/KM3NeTserver.md b/pages/KM3NeTserver.md
index de731f6..fb6699a 100644
--- a/pages/KM3NeTserver.md
+++ b/pages/KM3NeTserver.md
@@ -1,6 +1,7 @@
---
Title: KM3NeT server
Author: Jutta
+status: dump
---
* For all data not publishable through the IVOA, serving as interface and/or server to the data
diff --git a/pages/Multimessenger.md b/pages/Multimessenger.md
index f757ed7..ac525ba 100644
--- a/pages/Multimessenger.md
+++ b/pages/Multimessenger.md
@@ -4,4 +4,5 @@ Author: Feifei
Topics:
- alert types
- short description of the system
+status: unedited
---
diff --git a/pages/OSP.md b/pages/OSP.md
index fbab904..41088ed 100644
--- a/pages/OSP.md
+++ b/pages/OSP.md
@@ -4,6 +4,7 @@ Author: Jutta
Topics:
- installation
- interface through software
+status: dump
---
* Git project serving webpage (Gitbook)
diff --git a/pages/Procedures.md b/pages/Procedures.md
index db4ebd5..8ca343c 100644
--- a/pages/Procedures.md
+++ b/pages/Procedures.md
@@ -4,6 +4,7 @@ Author: Kay, Jutts
Topics:
- data/software releases
- publication procedures
+status: dump
---
## Establishing the Open Science Committee
diff --git a/pages/Processing.md b/pages/Processing.md
index c01bbc2..15ccbc1 100644
--- a/pages/Processing.md
+++ b/pages/Processing.md
@@ -1,6 +1,7 @@
---
Title: Processing overview
Author: Jutta
+status: dump
---
Data processing therefore follows a tier-based approach \cite{km3comp}, where initial
diff --git a/pages/Python.md b/pages/Python.md
index 263d86b..ebc6459 100644
--- a/pages/Python.md
+++ b/pages/Python.md
@@ -4,6 +4,7 @@ Authors: Tamas, Jutta
Topics:
- installation
- interface through software
+status: dump
---
diff --git a/pages/Quality.md b/pages/Quality.md
index 0ecb0fb..acd20bd 100644
--- a/pages/Quality.md
+++ b/pages/Quality.md
@@ -4,4 +4,5 @@ Author: Rodri
Topics:
- quality checks on event data
- quality indicators
+status: unedited
---
diff --git a/pages/Repositories_Zenodo.md b/pages/Repositories.md
similarity index 97%
rename from pages/Repositories_Zenodo.md
rename to pages/Repositories.md
index 1a2a2ff..e05549a 100644
--- a/pages/Repositories_Zenodo.md
+++ b/pages/Repositories.md
@@ -1,6 +1,7 @@
---
Title: Integrating to Zenodo
Author: Jutta
+status: dump
---
* For findability data needs to be citable and registered within a large repository (e.g. DataCite)
diff --git a/pages/Repositories_VOserver.md b/pages/Repositories_VOserver.md
deleted file mode 100644
index c4e9839..0000000
--- a/pages/Repositories_VOserver.md
+++ /dev/null
@@ -1,4 +0,0 @@
----
-Title: VO repository
-Author: Jutta
----
diff --git a/pages/ScienceTargets.md b/pages/ScienceTargets.md
index a331225..5e12eaa 100644
--- a/pages/ScienceTargets.md
+++ b/pages/ScienceTargets.md
@@ -1,6 +1,7 @@
---
Title: Scientific targets
Author: Jannik
+status: draft
---
# Scientific Targets
diff --git a/pages/SeaScience.md b/pages/SeaScience.md
index 08e416d..2ad2052 100644
--- a/pages/SeaScience.md
+++ b/pages/SeaScience.md
@@ -4,7 +4,7 @@ Author: Jannik
Topics:
- options for sea science
- types of environmental data
-Status: draft
+status: draft
---
## Environmental data
diff --git a/pages/Simulation.md b/pages/Simulation.md
index 1b6d316..2a1c62b 100644
--- a/pages/Simulation.md
+++ b/pages/Simulation.md
@@ -1,6 +1,7 @@
---
Title: Event simulation
Author: Jutta, Tamas
+status: dump
---
In order to evaluate and test physical models of neutrino interactions and productions, a huge variety of simulations need to be performed and compared to the data taken with the KM3NeT detectors.
diff --git a/pages/Softwaredevelopment.md b/pages/Softwaredevelopment.md
index b654851..701b7bd 100644
--- a/pages/Softwaredevelopment.md
+++ b/pages/Softwaredevelopment.md
@@ -4,4 +4,5 @@ Author: Tamas
Topics:
- using the python template
- software development recommendations
+status: unedited
---
diff --git a/pages/Usecase_ANTARES.md b/pages/Usecase_ANTARES.md
index 4eef6d4..3bb87ea 100644
--- a/pages/Usecase_ANTARES.md
+++ b/pages/Usecase_ANTARES.md
@@ -5,4 +5,5 @@ Topics:
- short description analysis
- data description
- linking to notebooks & background
+status: unedited
---
diff --git a/pages/Usecase_Environmental.md b/pages/Usecase_Environmental.md
index 3520f9a..29e3c0a 100644
--- a/pages/Usecase_Environmental.md
+++ b/pages/Usecase_Environmental.md
@@ -4,6 +4,7 @@ Topics:
- short description use
- data description
- interfaces
+status: dump
---
* Acoustic data will be provided through separate REST-API
diff --git a/pages/Usecase_KM3NeT.md b/pages/Usecase_KM3NeT.md
index 4bdbcc9..cf9ddbf 100644
--- a/pages/Usecase_KM3NeT.md
+++ b/pages/Usecase_KM3NeT.md
@@ -5,6 +5,7 @@ Topics:
- short description analysis
- data description
- linking to notebooks & background
+status: unedited
---
diff --git a/pages/Using_alerts.md b/pages/Using_alerts.md
index a323001..6b8eb79 100644
--- a/pages/Using_alerts.md
+++ b/pages/Using_alerts.md
@@ -4,8 +4,11 @@ Author: Feifei, Jutta
Topics:
- how to get the alerts?
- Description of VOEvent
+status: unedited
+Comment: |
+ To create such a page from the notebook,
+ export the jupyter notebook as markdown
+ and add to this text file and store the notebook
+ in the notebooks/ folder
---
-To create such a page from the notebook,
-* export the jupyter notebook as markdown and add to this text file
-* store the notebook in the notebooks/ folder
diff --git a/pages/Using_km3events.md b/pages/Using_km3events.md
index 8bc1805..9d2b3d3 100644
--- a/pages/Using_km3events.md
+++ b/pages/Using_km3events.md
@@ -4,8 +4,11 @@ Author: Steffen, Jutta
Topics:
- describe download
- notebook
+status: unedited
+Comment: |
+ To create such a page from the notebook,
+ export the jupyter notebook as markdown
+ and add to this text file and store the notebook
+ in the notebooks/ folder
---
-To create such a page from the notebook,
-* export the jupyter notebook as markdown and add to this text file
-* store the notebook in the notebooks/ folder
diff --git a/pages/Using_plotdata.md b/pages/Using_plotdata.md
index 2f824ab..e17e67a 100644
--- a/pages/Using_plotdata.md
+++ b/pages/Using_plotdata.md
@@ -4,8 +4,12 @@ Author: Rebecca, Jutta
Topics:
- ANTARES notebook example
- openkm3 example
+status: unedited
+Comment: |
+ To create such a page from the notebook,
+ export the jupyter notebook as markdown
+ and add to this text file and store the notebook
+ in the notebooks/ folder
---
-To create such a page from the notebook,
-* export the jupyter notebook as markdown and add to this text file
-* store the notebook in the notebooks/ folder
+
diff --git a/pages/Using_templates.md b/pages/Using_templates.md
index caa38ca..fc48ee4 100644
--- a/pages/Using_templates.md
+++ b/pages/Using_templates.md
@@ -6,4 +6,5 @@ Topics:
- guidelines
- git analysis template description
- guidelines
+status: unedited
---
diff --git a/pages/Using_vo.md b/pages/Using_vo.md
index ae8cea6..035e483 100644
--- a/pages/Using_vo.md
+++ b/pages/Using_vo.md
@@ -5,7 +5,7 @@ Topics:
- summary from open course
- https://edu.km3net.de/course/accessing-the-km3net-open-data/
-Status: draft
+status: dump
---
# Accessing data through the VO server
diff --git a/pages/VOserver.md b/pages/VOserver.md
index 5cf2df8..498a344 100644
--- a/pages/VOserver.md
+++ b/pages/VOserver.md
@@ -1,23 +1,21 @@
---
Title: VOserver
Author: Jutta
+status: unedited
---
-## Approach to the Virtual Observatory standard
+## The Virtual Observatory server
-The Virtual Observatory (VO) standards\cite{VO}
-serve to create an interface between astronomy-related data resources
-from astrophysics experiments which act as data providers. The focus is on the scientific end user to easily interface from their personal
-computer with the provided data sets. The KM3NeT collaboration is a data
-provider to the VO and operates a data server\footnote{at \url{http://vo.km3net.de/}} running the DaCHS software\cite{dachs}.
+## Publication
+
+The Virtual Observatory (VO) standards serve to create an interface between astronomy-related data resources from astrophysics experiments which act as data providers. The focus is on the scientific end user to easily interface from their personal
+computer with the provided data sets. The KM3NeT collaboration is a data provider to the VO and operates a data server at http://vo.km3net.de/ running the (DaCHS software)[http://docs.g-vo.org/DaCHS/].
The well-developed data sharing regime of the VO serves well as a
guideline for the implementation of astrophysical data sharing in the
-KM3NeT collaboration. However, considering the role of neutrino physics
-just at the edge of use for astronomical studies, KM3NeT data
-integration also meets some limitations considering the scientific
-usability of the provided data sets. In addition to that, publication of
-data through VO standards is clearly limited to astronomy-related data
-in a celestial reference frame.
+KM3NeT collaboration.
+
+## Data access
+Tabulated high-level neutrino event data can be provided through the VO registry, utilizing access protocols like the Table Access Protocol (TAP) and query languages like the Astronomical Data Query Language (ADQL). To query these data sets related to astronomical sources, the Simple Cone Search (SCS) protocol allows to pick specific events according to particle source direction.
* Running server with DaCHS software
* Registered as data provider to the VO (ivo://km3net.org)
diff --git a/pages/Workflows.md b/pages/Workflows.md
index 8fdd493..131c5c0 100644
--- a/pages/Workflows.md
+++ b/pages/Workflows.md
@@ -1,6 +1,7 @@
---
Title: Workflow management
Author: Jutta, Tamas
+status: dump
---
The simulation and raw data processing chains consist of workflows that are
--
GitLab