From a0f186f3d7fe88b70a4ed513f072f92b36dd78f3 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 9 Jul 2026 16:11:17 +0200 Subject: [PATCH 1/8] fix: remove duplicated changelog entry --- CHANGELOG.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2bed5..02190c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,16 +44,6 @@ tags that are defined in the server-configuration (the default is False). -# 6.3.0 (2026-06-29) - -## Changes - -- Automated annotation adding in write-record-endpoints has been removed. - The new query parameter `add_submission_tag` is added to write-record-endpoints. - If set to True, the server will add an automated annotation with the annotation - tags that are defined in the server-configuration (the default is False). - - # 6.2.2 (2026-06-26) ## Bugfixes -- 2.52.0 From c4d2003a15dec806ccf8bb3ada358e0d148312a7 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 9 Jul 2026 16:13:14 +0200 Subject: [PATCH 2/8] docs: start documentation --- docs/Makefile | 4 +- docs/authentication.rst | 208 +++++++++++++++++++++++++++++++++ docs/backends.rst | 97 ++++++++++++++++ docs/changelog.rst | 5 + docs/commands.rst | 218 ++++++++++++++++++++++++++++++++++ docs/conf.py | 38 ++++++ docs/configuration.rst | 234 +++++++++++++++++++++++++++++++++++++ docs/endpoints.rst | 252 ++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 34 +++++- docs/installation.rst | 49 ++++++++ docs/introduction.rst | 62 ++++++++++ docs/quickstart.rst | 71 +++++++++++ docs/roman_numerals.py | 46 ++++++++ pyproject.toml | 2 + 14 files changed, 1315 insertions(+), 5 deletions(-) create mode 100644 docs/authentication.rst create mode 100644 docs/backends.rst create mode 100644 docs/changelog.rst create mode 100644 docs/commands.rst create mode 100644 docs/configuration.rst create mode 100644 docs/endpoints.rst create mode 100644 docs/installation.rst create mode 100644 docs/introduction.rst create mode 100644 docs/quickstart.rst create mode 100644 docs/roman_numerals.py diff --git a/docs/Makefile b/docs/Makefile index 973b097..9496490 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,8 +3,8 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= --fail-on-warning -SPHINXBUILD ?= sphinx-build +SPHINXOPTS ?= +SPHINXBUILD ?= /home/cristian/Develop/dump-things-service/.venv/bin/sphinx-build SOURCEDIR = . BUILDDIR = _build diff --git a/docs/authentication.rst b/docs/authentication.rst new file mode 100644 index 0000000..90abbb4 --- /dev/null +++ b/docs/authentication.rst @@ -0,0 +1,208 @@ +Authentication & Authorization +================================ + +The service uses *authentication sources* to authenticate and authorize clients. +Each collection can have its own list of authentication sources. + +Authentication Sources +----------------------- + +Authentication sources are tried in the order they are listed for a collection. +If an authentication source authenticates the token successfully, no further +sources are queried. If no source is defined for a collection, the +``config``-source is used. + +If the same authentication source is listed more than once, the duplicate entries +are ignored and a warning ``Ignoring duplicate authentication provider...`` is +logged. + +Available Sources +^^^^^^^^^^^^^^^^^ + +``config`` + Reads token information directly from the configuration file. + This is the default source. + +``forgejo`` + Uses a `Forgejo `_ instance to authenticate tokens + and to determine the user ID and the incoming label. + +Configuring Authentication Sources +------------------------------------ + +Authentication sources are configured per collection via the ``auth_sources`` +key: + +.. code-block:: yaml + + collections: + my_collection: + default_token: anon_read + curated: my_collection/curated + auth_sources: + - type: forgejo + url: https://forgejo.example.com/api/v1 + organization: data_handling + team: data_entry_personal + label_type: team + repository: reference-repository # optional + - type: config + +Config-based Authentication +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +No extra keys are required beyond ``type: config``. + +.. code-block:: yaml + + auth_sources: + - type: config + +Forgejo-based Authentication +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Forgejo source uses the Forgejo REST API to validate tokens and determine +permissions. The token must have at least these Forgejo API permissions: + +- **User** (Read): required to get user email and login name. +- **Organization** (Read): required to determine team membership. +- **Repository** (Read): required only when ``repository`` is set. + +.. list-table:: Forgejo source configuration keys + :header-rows: 1 + :widths: 25 10 65 + + * - Key + - Required + - Description + * - ``url`` + - ✓ + - API URL of the Forgejo instance, e.g. ``https://forge.example.com/api/v1``. + * - ``organization`` + - ✓ + - Name of the Forgejo organization. + * - ``team`` + - ✓ + - Name of the team inside the organization. + * - ``label_type`` + - ✓ + - ``team`` or ``user``. Determines the incoming label format (see below). + * - ``repository`` + - ✗ + - If set, the token is only authorized if the team has access to this repository. + * - ``instance_id`` + - ✗ + - Optional identifier to disambiguate users on different Forgejo instances. Defaults to a hash of ``url``. + +Incoming Label Generation +'''''''''''''''''''''''''' + +The ``label_type`` key controls how the incoming label is derived: + +- ``team`` → ``forgejo-team--`` +- ``user`` → ``forgejo-user-`` + +Permission Mapping +'''''''''''''''''' + +Forgejo team permissions are mapped to service permissions as follows: + +.. list-table:: Mapping of ``repo.code`` to service permissions + :header-rows: 1 + + * - ``repo.code`` + - curated_read + - incoming_read + - incoming_write + - curated_right + - zones_access + * - ``none`` + - ✗ + - ✗ + - ✗ + - ✗ + - ✗ + * - ``read`` + - ✓ + - ✓ + - ✗ + - ✗ + - ✗ + * - ``write`` + - ✓ + - ✓ + - ✓ + - ✗ + - ✗ + +.. list-table:: Mapping of ``repo.actions`` to service permissions + :header-rows: 1 + + * - ``repo.actions`` + - curated_read + - incoming_read + - incoming_write + - curated_right + - zones_access + * - ``none`` + - ✗ + - ✗ + - ✗ + - ✗ + - ✗ + * - ``read`` + - ✗ + - ✗ + - ✗ + - ✗ + - ✗ + * - ``write`` + - ✓ + - ✓ + - ✓ + - ✓ + - ✓ + +Access Modes +------------- + +Token permissions for a collection are expressed as a *mode*: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Mode + - Granted permissions + * - ``NOTHING`` + - No access at all. + * - ``READ_CURATED`` + - Read curated data only. + * - ``READ_INCOMING`` + - Read own incoming zone. + * - ``WRITE_INCOMING`` + - Read & write own incoming zone; read curated. + * - ``WRITE_COLLECTION`` + - Full access: read curated + read/write own incoming zone. + * - ``CURATOR`` + - Full access plus read/write all incoming zones and curated. + +Administrator Tokens +--------------------- + +Administrator tokens are required for the administration endpoints (creating / +deleting collections, tokens, etc.). They are **always** stored as SHA-256 +hashes. To create one: + +.. code-block:: bash + + # Compute the hash + dump-things-hash-token 'my-admin-secret' + + # Pass the hash when starting the service + dump-things-service /path/to/storage \ + --admin-token-hash <64-digit hex hash> + +Alternatively, set ``DTS_ADMIN_TOKEN`` to the plain-text token in the +environment before starting the service. The command-line option takes +precedence if both are provided. diff --git a/docs/backends.rst b/docs/backends.rst new file mode 100644 index 0000000..5934541 --- /dev/null +++ b/docs/backends.rst @@ -0,0 +1,97 @@ +Storage Backends +================ + +The service supports multiple storage backends. A backend can be configured +per collection in the configuration file. If no backend is defined for a +collection, ``record_dir+stl`` is used by default. + +Available Backends +------------------ + +``record_dir`` + Stores records as YAML files in a directory structure defined by + `Dump Things Storage v0 `_. + Reads the backend configuration from a *record collection configuration file* + as described in the same specification. + +``record_dir+stl`` *(default)* + Same as ``record_dir``, but adds a **schema-type layer** (STL): + + - ``schema_type`` attributes are stripped from the top-level mapping of a + record before it is stored. + - When a record is read back, a ``schema_type`` attribute is re-inserted + using the schema to determine the correct class URI. + + This is useful when endpoints return records of multiple classes, because + clients can determine the class of each record from the ``schema_type`` + attribute. + +``sqlite`` + Stores records in a SQLite database. One database file named + ``__sqlite-records.db`` is created per curated/incoming area. + +``sqlite+stl`` + Same as ``sqlite``, but with the same schema-type layer as + ``record_dir+stl``. + +.. note:: + + All backends can co-exist independently in the same directory. There are no + file-name collisions between backends, as long as no class name starts with + ``.`` or ``_``. + +Configuring a Backend +--------------------- + +Specify the backend in the collection configuration with the ``backend`` key: + +.. code-block:: yaml + + collections: + my_collection: + schema: https://example.org/schema.yaml + default_token: anon_read + curated: my_collection/curated + + backend: + type: record_dir+stl # explicit default + +.. code-block:: yaml + + collections: + another_collection: + schema: https://example.org/schema.yaml + default_token: anon_read + curated: another_collection/curated + + backend: + type: sqlite + +Migrating Between Backends +--------------------------- + +Use ``dump-things-copy-store`` to copy data between backends +(see :doc:`commands`). + +Example — ``record_dir`` → ``sqlite``: + +.. code-block:: bash + + dump-things-copy-store \ + record_dir:/path/to/data/curated \ + sqlite:/path/to/data/curated + +Example — ``record_dir+stl`` → ``sqlite`` (schema required): + +.. code-block:: bash + + dump-things-copy-store \ + --schema https://concepts.inm7.de/s/flat-data/unreleased.yaml \ + record_dir+stl:/path/to/data/curated \ + sqlite:/path/to/data/curated + +.. note:: + + When copying from a ``record_dir`` or ``record_dir+stl`` backend that was + modified outside the service (e.g. by manually adding/removing files), run + ``dump-things-rebuild-index`` first to ensure the index is up to date. diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..08efb85 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,5 @@ +Changelog +========= + +.. include:: ../CHANGELOG.md + :parser: myst_parser.sphinx_ diff --git a/docs/commands.rst b/docs/commands.rst new file mode 100644 index 0000000..9362ad0 --- /dev/null +++ b/docs/commands.rst @@ -0,0 +1,218 @@ +Command-Line Tools +================== + +The package installs the following command-line tools. + +Service Commands +---------------- + +dump-things-service +^^^^^^^^^^^^^^^^^^^ + +Start the Dump Things Service. + +.. code-block:: text + + dump-things-service [OPTIONS] + +**Arguments** + +```` + (Mandatory) Path to the directory used as anchor for all relative paths in + the configuration. Unless ``-c/--config`` is provided, the service looks for + a configuration file at ``/.dumpthings.yaml``. + +**Options** + +.. option:: --host + + IP address on which the service should accept connections. + Default: ``0.0.0.0``. + +.. option:: --port + + Port on which the service should accept connections. Default: ``8000``. + +.. option:: -c, --config + + Path to a configuration file. If no persisted configuration is found in the + data store, the service is initialized with the content of this file. + Useful for migrating from service version 5 and lower to version 6. + +.. option:: --origins + + Add a CORS origin host. Repeat to add multiple CORS origin URLs. + +.. option:: --root-path + + Set the ASGI ``root_path`` for applications mounted below a given URL path. + +.. option:: --log-level + + Log level. Allowed values: ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. + Default: ``WARNING``. + +.. option:: --admin-token-hash + + Set an administrator token hash (64-digit hexadecimal SHA-256 hash of the + plain-text token). If not provided, the plain-text token is read from the + environment variable ``DTS_ADMIN_TOKEN``. + +.. option:: --ignore-default-config-file + + If set, the service will not try to initialize itself from the default + configuration file ``/.dumpthings.yaml``. + +**Example** + +.. code-block:: bash + + dump-things-service /data/store --host 127.0.0.1 --port 8000 + +Configuration Tools +------------------- + +dump-things-upload-config +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Read a YAML configuration file and apply it to a running service via the +administration endpoints. + +.. code-block:: text + + dump-things-upload-config [OPTIONS] + +**Options** + +.. option:: --server + + Base URL of the running service (e.g. ``http://127.0.0.1:8000``). + +.. option:: --admin-token + + Plain-text administrator token. + +dump-things-download-config +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Read the current configuration from a running service and write it to a file. + +.. code-block:: text + + dump-things-download-config [OPTIONS] + +**Options** + +.. option:: --server + + Base URL of the running service. + +.. option:: --admin-token + + Plain-text administrator token. + +Maintenance Tools +----------------- + +dump-things-rebuild-index +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rebuild the persistent index of a ``record_dir`` store. Run this after the +store was modified outside the service (e.g. by manually adding/removing +files). + +.. code-block:: bash + + dump-things-rebuild-index + +dump-things-copy-store +^^^^^^^^^^^^^^^^^^^^^^^ + +Copy a collection from a source store to a destination store. The source and +destination are specified as ``:`` where ```` is one of +``record_dir``, ``record_dir+stl``, ``sqlite``, or ``sqlite+stl``. + +.. code-block:: bash + + dump-things-copy-store [--schema ] + +.. option:: -s, --schema + + Required when either backend is ``record_dir+stl`` or ``sqlite+stl``. + +**Examples** + +.. code-block:: bash + + # record_dir → sqlite + dump-things-copy-store \ + record_dir:/path/to/curated \ + sqlite:/path/to/curated + + # record_dir+stl → sqlite (schema required) + dump-things-copy-store \ + --schema https://concepts.inm7.de/s/flat-data/unreleased.yaml \ + record_dir+stl:/path/to/curated \ + sqlite:/path/to/curated + +dump-things-pid-check +^^^^^^^^^^^^^^^^^^^^^^ + +Check all PIDs in a store to verify they can be resolved (useful after +schema changes that affect prefix definitions). + +.. code-block:: bash + + dump-things-pid-check + +dump-things-create-merged-schema +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create a fully self-contained schema that statically embeds all schemas +imported by the original schema. + +.. code-block:: bash + + dump-things-create-merged-schema + +dump-things-hash-token +^^^^^^^^^^^^^^^^^^^^^^^^ + +Compute the SHA-256 hash of a plain-text token for use with +``--admin-token-hash``. + +.. code-block:: bash + + dump-things-hash-token + +Audit Tools +----------- + +dump-things-gitaudit-report +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Show the audit log for all PIDs matching a pattern. + +.. code-block:: bash + + dump-things-gitaudit-report + +```` is a Python ``re``-module pattern. +Use ``'.*'`` to report changes for all PIDs. + +Each log entry contains: + +- Timestamp of the change. +- Curator ID. +- Diff of the change. +- The resulting record. + +dump-things-gitaudit-rebuild-index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rebuild the index for a ``gitaudit`` backend. Normally not required in +regular operations; useful for maintenance. + +.. code-block:: bash + + dump-things-gitaudit-rebuild-index diff --git a/docs/conf.py b/docs/conf.py index b79be80..478969f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,6 +3,14 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('..')) + # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information @@ -21,6 +29,9 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx_autodoc_typehints', 'sphinx.ext.viewcode', + 'sphinx.ext.intersphinx', + 'sphinx.ext.napoleon', + 'myst_parser', ] templates_path = ['_templates'] @@ -29,9 +40,30 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] primary_domain = 'py' autoclass_content = "both" +# autodoc settings +autodoc_default_options = { + 'members': True, + 'undoc-members': True, + 'show-inheritance': True, +} +autosummary_generate = True + typehints_use_signature = True typehints_use_signature_return = True +# Napoleon settings (Google/NumPy style docstrings) +napoleon_google_docstring = True +napoleon_numpy_docstring = True + +# MyST settings (Markdown support) +myst_enable_extensions = ['colon_fence'] + +# Intersphinx mapping +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'pydantic': ('https://docs.pydantic.dev/latest/', None), +} + # we build some docstrings from loguru. define some no-op substitutions # to avoid errors rst_prolog = """ @@ -46,3 +78,9 @@ rst_prolog = """ html_theme = 'sphinx_rtd_theme' html_static_path = ['_static'] + +# RTD theme options +html_theme_options = { + 'navigation_depth': 4, + 'titles_only': False, +} diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 0000000..7b00a00 --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,234 @@ +Configuration +============= + +The service is configured via a YAML configuration file. The tool +``dump-things-upload-config`` can load a configuration from a file and apply +it to a running service via the administration endpoints. + +Configuration File Structure +----------------------------- + +A configuration file (version 2) has the following top-level structure: + +.. code-block:: yaml + + type: collections # must be "collections" + version: 2 # must be 2 + + collections: + : + ... + + tokens: + : + ... + +Collections +----------- + +Each entry under ``collections`` defines a collection. The following keys are +supported: + +``schema`` (required) + URL of the `LinkML `_ schema for this collection. + +``default_token`` (required) + Name of the token to use when no token is provided by a client. All + permissions in the default token are OR-ed with any client-provided token. + +``curated`` (required) + Path to the curated area of the collection. A relative path is interpreted + relative to the storage root. + +``incoming`` (optional) + Path to the incoming area. Required if the collection should accept uploads. + +``backend`` (optional) + Storage backend configuration. If omitted, ``record_dir+stl`` is used. + See :doc:`backends` for details. + +``use_classes`` (optional) + List of LinkML class names that should receive store/validate endpoints. + All other classes defined in the schema are ignored. + +``ignore_classes`` (optional) + List of LinkML class names to exclude from store/validate endpoints. + +``auth_sources`` (optional) + List of authentication source configurations. See :doc:`authentication`. + +``submission_tags`` (optional) + Override the default annotation tags. See `Submission Annotation Tags`_ below. + +``audit_backends`` (optional) + List of audit-backend configurations. See `Audit Backends`_ below. + +Example +^^^^^^^ + +.. code-block:: yaml + + collections: + personal_records: + default_token: no_access + schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml + curated: curated/personal_records + incoming: /tmp/personal_records/incoming + use_classes: + - Organization + - Person + - Project + +Tokens +------ + +Each entry under ``tokens`` defines a token. The following keys are supported: + +``representation`` (required) + The plain-text value the client must supply in the ``X-DumpThings-Token`` + HTTP header. If ``hashed`` is ``True`` this must be the SHA-256 hex digest + of the plain-text token. + +``hashed`` (optional, default: ``False``) + If ``True``, the representation is treated as a SHA-256 hash. + Use ``dump-things-hash-token`` to compute the correct hash. + +``user_id`` (required) + An identifier that is added as an annotation to each uploaded record. + +``collections`` (required) + Mapping from collection name to per-collection rights. + +Per-collection rights +^^^^^^^^^^^^^^^^^^^^^ + +Each collection entry inside a token has: + +``mode`` + Access mode. Available modes: + + - ``NOTHING`` – no access. + - ``READ_CURATED`` – read curated data only. + - ``READ_INCOMING`` – read own incoming zone. + - ``WRITE_INCOMING`` – read & write own incoming zone, read curated. + - ``WRITE_COLLECTION`` – full access (read curated + read/write incoming). + - ``CURATOR`` – curator-level access (curated read/write + all incoming). + +``incoming_label`` + Label that defines the zone in the incoming area for this token/collection. + Multiple tokens can share the same label, which means they share the zone. + +Example +^^^^^^^ + +.. code-block:: yaml + + tokens: + basic_access: + representation: anonymous + hashed: False + user_id: anonymous_user + collections: + rooms_and_buildings: + mode: READ_CURATED + incoming_label: '' + + contributor_bob: + representation: bob + user_id: Bob + collections: + rooms_and_buildings: + mode: WRITE_COLLECTION + incoming_label: new_rooms_and_buildings + +Reserved Collection Names +-------------------------- + +The following collection names are reserved and must **not** be used: + +- ``collections`` +- ``tokens`` +- ``admin_tokens`` +- ``__dump_things__`` + +Submission Annotation Tags +--------------------------- + +By default the service annotates uploaded records with: + +- Submitter ID: ``http://purl.obolibrary.org/obo/NCIT_C54269`` +- Submission time: ``http://semanticscience.org/resource/SIO_001083`` + +Override these defaults per collection with ``submission_tags``: + +.. code-block:: yaml + + collections: + collection_1: + ... + submission_tags: + submitter_id_tag: schema:user_id + submission_time_tag: schema:time + +Audit Backends +-------------- + +The service supports audit logs of changes made via the curation interface. +Configure them per collection with ``audit_backends``: + +.. code-block:: yaml + + collections: + collection_1: + ... + audit_backends: + - type: gitaudit + path: /path/to/audit-log-dir + auto_flush_timeout: 60 # seconds (default: 60, minimum: 1) + +The ``gitaudit`` backend stores changes in a bare Git repository at ``path``. +After ``auto_flush_timeout`` seconds without new entries the current changeset +is committed. + +Use ``dump-things-gitaudit-report`` to display the audit log and +``dump-things-gitaudit-rebuild-index`` to rebuild its index (see :doc:`commands`). + +Tips & Tricks +------------- + +Same Backend for Incoming and Curated Areas +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can configure the service so that incoming records are immediately visible +in the curated area by setting the final incoming path to equal the curated +path: + +.. code-block:: yaml + + collections: + datamgt: + default_token: anon_read + curated: datamgt/curated + incoming: datamgt + + tokens: + trusted-submitter-token: + user_id: trusted_submitter + collections: + datamgt: + mode: WRITE_COLLECTION + incoming_label: curated # datamgt/curated == curated path + +Migrating from Version 5 (or Lower) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use the ``-c/--config`` option on first start to initialise the persisted +configuration from an existing file. Once started, the configuration is stored +in the internal database and the ``-c`` option can be omitted on subsequent +starts: + +.. code-block:: bash + + dump-things-service /path/to/storage -c /path/to/old-config.yaml + +See also the migration guide in the :doc:`changelog`. diff --git a/docs/endpoints.rst b/docs/endpoints.rst new file mode 100644 index 0000000..5bee715 --- /dev/null +++ b/docs/endpoints.rst @@ -0,0 +1,252 @@ +HTTP Endpoints +============== + +All endpoints are accessible via the interactive Swagger UI at ``/docs`` after +starting the service. + +Most endpoints that operate on records require a *collection name* and a valid +token supplied in the ``X-DumpThings-Token`` HTTP header. + +.. contents:: Endpoint overview + :local: + :depth: 1 + +User Endpoints +-------------- + +Store a Record +^^^^^^^^^^^^^^ + +``POST //record/`` + +Stores an object of type ```` (defined in the collection schema) in the +incoming area for the authenticated token's zone. + +- **Requires**: write permission (``WRITE_INCOMING`` or higher). +- **Query parameters**: + + - ``format``: ``json`` (default) or ``ttl`` (Turtle RDF). + - ``add_submission_tag``: if ``true`` the server adds automated submitter-ID + and submission-time annotations. Default: ``false``. + +- **Content-Type**: ``application/json`` for JSON; ``text/turtle`` for Turtle. +- **Returns**: list of all stored records (may contain more than one entry when + the posted object contains inlined records). + +Validate a Record +^^^^^^^^^^^^^^^^^ + +``POST //validate/record/`` + +Validates a record without storing it. Supports the same parameters as +``POST //record/``. + +Retrieve All Records of a Class +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``GET //records/`` + +Returns all readable records of type ```` (and its subclasses). +Records from the incoming zone take precedence over identical PIDs in the +curated area. + +- **Requires**: read permission. +- **Query parameters**: + + - ``format``: ``json`` (default) or ``ttl``. + - ``matching``: wildcard pattern for ``sqlite`` backends; ``%`` matches any + characters (case-insensitive). Ignored by ``record_dir`` backends. + +Retrieve All Records of a Class (Paginated) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``GET //records/p/`` + +Same as above but with pagination support. + +- **Additional query parameters**: + + - ``page``: page number (starting at 1). + - ``size``: records per page (default: 50). + +- **Response structure**: + + .. code-block:: json + + { + "items": [""], + "total": 123, + "page": 1, + "size": 50, + "pages": 3 + } + +Retrieve All Records +^^^^^^^^^^^^^^^^^^^^^ + +``GET //records/`` + +Returns all readable records of a collection regardless of class. +Supports the same ``format`` and ``matching`` parameters. + +Retrieve All Records (Paginated) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``GET //records/p/`` + +Paginated version of the endpoint above; supports the same ``page`` and +``size`` parameters. + +Retrieve a Single Record by PID +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``GET //record?pid=`` + +Retrieves the record with the given PID. If both the incoming zone and the +curated area contain a record with the same PID, the incoming record is +returned. + +- **Query parameters**: + + - ``format``: ``json`` (default) or ``ttl``. + +Delete a Record +^^^^^^^^^^^^^^^ + +``DELETE //record?pid=`` + +Deletes the record with PID ```` from the **incoming** area. + +- **Requires**: write permission. +- **Returns**: ``true`` if the record was deleted, ``false`` otherwise. + +Server Information +^^^^^^^^^^^^^^^^^^ + +``GET /server`` + +Returns information about the running service: + +.. code-block:: json + + { + "version": "", + "collections": [ + { + "name": "collection_1", + "schema": "https://example.org/schema.yaml", + "classes": ["Thing", "Agent", "Person"] + } + ] + } + +Maintenance Mode +^^^^^^^^^^^^^^^^ + +``POST /maintenance`` + +Puts a collection into maintenance mode. In maintenance mode only tokens with +curator privileges can access the collection. + +Request body: + +.. code-block:: json + + { + "collection": "collection_1", + "active": true + } + +Curation Endpoints +------------------- + +Curation endpoints allow direct read/write access to the curated area and all +incoming zones. A ``CURATOR`` token is required. + +Refer to the interactive API documentation at ``/docs`` on a running service +for the full list of curation endpoints. + +Administration Endpoints +-------------------------- + +Administration endpoints require an **administrator token**. Use +``dump-things-upload-config`` to generate the necessary POST requests from a +configuration file (see :doc:`commands`). + +Collections +^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 10 30 60 + + * - Method + - Path + - Description + * - POST + - ``/collections`` + - Create a new collection. + * - GET + - ``/collections`` + - List all collections. + * - GET + - ``/collections/`` + - Get information about a specific collection. + * - PUT + - ``/collections/`` + - Update an existing collection. + * - DELETE + - ``/collections/`` + - Remove a collection from the service state (data is not deleted). + +Tokens +^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 10 30 60 + + * - Method + - Path + - Description + * - POST + - ``/tokens`` + - Create a new token. + * - GET + - ``/tokens`` + - List all tokens (including representations). + * - GET + - ``/tokens/`` + - Get information about a specific token. + * - PUT + - ``/tokens/`` + - Update an existing token. + * - DELETE + - ``/tokens/`` + - Delete a token. + +Admin Tokens +^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 10 30 60 + + * - Method + - Path + - Description + * - POST + - ``/admin_tokens`` + - Create a new admin token (representation must be a SHA-256 hash). + * - GET + - ``/admin_tokens`` + - List all admin tokens. + * - GET + - ``/admin_tokens/`` + - Get information about a specific admin token. + * - PUT + - ``/admin_tokens/`` + - Update an existing admin token. + * - DELETE + - ``/admin_tokens/`` + - Delete an admin token. diff --git a/docs/index.rst b/docs/index.rst index 17022f1..13d0ab1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,10 +1,38 @@ -The `dump-thing-server` documentation -===================================== +Dump Things Service Documentation +=================================== -HERE BE CONTENT... +.. image:: https://badge.fury.io/py/dump-things-service.svg + :target: https://pypi.python.org/pypi/dump-things-service/ + :alt: PyPI version + +*Dump Things Service* is a FastAPI-based HTTP service for storing and retrieving +schema-conform linked-data records. It supports multiple storage backends, +flexible authentication sources, and a curation workflow for distributed data +acquisition. + +.. toctree:: + :maxdepth: 2 + :caption: User Guide + + introduction + installation + quickstart + configuration + backends + authentication + endpoints + commands + +.. toctree:: + :maxdepth: 1 + :caption: Developer Reference + + api/index + changelog Indices and tables ================== * :ref:`genindex` +* :ref:`modindex` * :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..78af667 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,49 @@ +Installation +============ + +Requirements +------------ + +- Python 3.11 or later +- ``sqlite3`` (usually bundled with Python) + +Installing from PyPI +--------------------- + +The service is available on PyPI and can be installed with ``pip``: + +.. code-block:: bash + + pip install dump-things-service + +Installing for Development +--------------------------- + +Clone the repository and install the package with all development dependencies +using `hatch `_: + +.. code-block:: bash + + git clone https://codeberg.org/datalink/dump-things-server.git + cd dump-things-server + pip install hatch + hatch env create + +Building the Documentation +--------------------------- + +The documentation dependencies are declared in the ``docs`` optional-dependency +group. To install them and build the HTML documentation: + +.. code-block:: bash + + pip install "dump-things-service[docs]" + make -C docs html + +Alternatively, if you are using ``hatch``: + +.. code-block:: bash + + hatch run docs:build + +The generated HTML will be placed in ``docs/_build/html/``. diff --git a/docs/introduction.rst b/docs/introduction.rst new file mode 100644 index 0000000..20b3eae --- /dev/null +++ b/docs/introduction.rst @@ -0,0 +1,62 @@ +Introduction +============ + +What is Dump Things Service? +----------------------------- + +*Dump Things Service* is an implementation of a service that allows storing +and retrieving linked data. It builds the backend for linked-data applications. +Linked-data applications can retrieve data as JSON records or as +`TTL `_ (Turtle) documents from a Dump Things +Service instance. Multiple *Dump Things Service* instances currently serve as +backends for `shacl-vue `_. + +All data items stored on a *Dump Things Service* must be structured according to +a defined schema. Schemata are defined in `LinkML `_. +The service supports multiple *collections*. Each data item is stored in a +collection. Each collection has a name and an associated schema. The service +dynamically reads the respective schemata and creates validation code for data +items. This ensures that only items that adhere to the schema can be stored in +the respective collection. + +Curated Area and Incoming Area +-------------------------------- + +The *Dump Things Service* supports distributed data acquisition while still +controlling the quality of collection data. This is done by distinguishing +between two areas: + +- **Curated area**: represents the "official", high-quality, collection content. +- **Incoming area** (user-specific inboxes): where users propose new records or + changes to existing records. + +A curation workflow moves proposed changes from the incoming area into the +curated area. Curators — entities with specific privileges — can inspect +proposed changes, accept, modify, or reject them. Curation can be performed by +persons or by automated agents. The service provides tools and APIs to support +both options. + +Tokens and Permissions +----------------------- + +To submit a record to a collection, a **token** is required. Tokens carry: + +- Read and write permissions for the incoming areas of collections. +- Read permissions for the curated area of a collection. +- A submitter ID that is annotated to each submitted record. +- A token-specific **zone** in the incoming area. + +Multiple tokens can share the same zone, allowing multiple submitters to work +together on the same incoming area. + +Key Features +------------- + +- **Multiple storage backends**: ``record_dir``, ``record_dir+stl``, + ``sqlite``, and ``sqlite+stl``. +- **Flexible authentication**: config-file-based tokens and + `Forgejo `_-based authentication. +- **Audit logs**: track time, content, author, and curator for every change. +- **LinkML-based validation**: ensure data conforms to the defined schema. +- **GraphQL support**: query data via a GraphQL interface. +- **Pagination**: all list endpoints support optional pagination. diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000..52865a7 --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,71 @@ +Quick Start +=========== + +This guide walks you through starting the service with a minimal configuration. + +1. Start the Service +--------------------- + +After installation, start the service with: + +.. code-block:: bash + + dump-things-service /path/to/storage --host 127.0.0.1 --port 8000 + +Where ``/path/to/storage`` is a directory that will be used as the storage root. +Unless ``-c/--config`` is provided, the service looks for a configuration file +at ``/.dumpthings.yaml``. + +Alternatively, supply a configuration file explicitly: + +.. code-block:: bash + + dump-things-service /path/to/storage \ + --host 127.0.0.1 --port 8000 \ + -c /path/to/config.yaml + +2. Set an Administrator Token +------------------------------ + +An administrator token is required to create collections and tokens. +It can be provided as a SHA-256 hash on the command line: + +.. code-block:: bash + + # Generate a hash for the plain-text token "mysecret" + dump-things-hash-token 'mysecret' + # → <64-character hex string> + + # Start the service with the hashed admin token + dump-things-service /path/to/storage \ + --admin-token-hash <64-character hex string> + +Alternatively, set the environment variable ``DTS_ADMIN_TOKEN`` to the +plain-text token before starting the service: + +.. code-block:: bash + + export DTS_ADMIN_TOKEN=mysecret + dump-things-service /path/to/storage + +3. Upload a Configuration +-------------------------- + +The easiest way to configure collections and tokens is to write a YAML +configuration file (see :doc:`configuration`) and upload it with: + +.. code-block:: bash + + dump-things-upload-config \ + --server http://127.0.0.1:8000 \ + --admin-token mysecret \ + /path/to/config.yaml + +4. Explore the API +------------------ + +Once the service is running, open your browser at +``http://127.0.0.1:8000/docs`` to explore the interactive Swagger UI for all +available endpoints. + +For a full description of all endpoints see :doc:`endpoints`. diff --git a/docs/roman_numerals.py b/docs/roman_numerals.py new file mode 100644 index 0000000..0df0983 --- /dev/null +++ b/docs/roman_numerals.py @@ -0,0 +1,46 @@ +"""Compatibility shim for Sphinx's LaTeX writer. + +Some environments provide a broken or incomplete ``roman-numerals`` package +that does not expose the ``roman_numerals`` module expected by Sphinx. The +LaTeX writer only needs a small subset of the functionality: a ``RomanNumeral`` +class with ``to_lowercase()``. +""" + +from __future__ import annotations + + +class RomanNumeral: + def __init__(self, value: int) -> None: + if not isinstance(value, int): + raise TypeError('RomanNumeral expects an integer') + if value < 1: + raise ValueError('RomanNumeral expects a positive integer') + self.value = value + + def to_lowercase(self) -> str: + return _to_roman(self.value).lower() + + +def _to_roman(value: int) -> str: + numerals = [ + (1000, 'M'), + (900, 'CM'), + (500, 'D'), + (400, 'CD'), + (100, 'C'), + (90, 'XC'), + (50, 'L'), + (40, 'XL'), + (10, 'X'), + (9, 'IX'), + (5, 'V'), + (4, 'IV'), + (1, 'I'), + ] + result = [] + remaining = value + for arabic, roman in numerals: + while remaining >= arabic: + result.append(roman) + remaining -= arabic + return ''.join(result) diff --git a/pyproject.toml b/pyproject.toml index a62ee2b..111812c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ docs = [ "sphinx", "sphinx_rtd_theme", "sphinx_autodoc_typehints", + "myst-parser", ] [project.scripts] @@ -108,6 +109,7 @@ extra-dependencies = [ "sphinx", "sphinx_rtd_theme", "sphinx-autodoc-typehints", + "myst-parser", ] [tool.hatch.envs.docs.scripts] build = [ -- 2.52.0 From 6d1c44ceff106f744250611542d09b5182bf576f Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 14 Jul 2026 11:19:21 +0200 Subject: [PATCH 3/8] docs: improve introduction Tell the story of dump-things-service, add images to illustrate data-flow and work-flow. --- docs/configuration.rst | 94 +++- docs/images/dts-author-write.svg | 484 ++++++++++++++++++++ docs/images/dts-authors-reading.svg | 625 ++++++++++++++++++++++++++ docs/images/dts-curate.svg | 658 ++++++++++++++++++++++++++++ docs/images/dts-reader.svg | 411 +++++++++++++++++ docs/index.rst | 1 - docs/introduction.rst | 204 +++++++++ 7 files changed, 2473 insertions(+), 4 deletions(-) create mode 100644 docs/images/dts-author-write.svg create mode 100644 docs/images/dts-authors-reading.svg create mode 100644 docs/images/dts-curate.svg create mode 100644 docs/images/dts-reader.svg diff --git a/docs/configuration.rst b/docs/configuration.rst index 7b00a00..ed55499 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,10 +1,94 @@ Configuration ============= -The service is configured via a YAML configuration file. The tool -``dump-things-upload-config`` can load a configuration from a file and apply -it to a running service via the administration endpoints. +General Concepts +---------------- + +A Dump Things Service instance is created by executing the command +``dump-things-service`` on the command line. The command has one mandatory +argument, the path to the **storage root**. This is a path to a locally accessible +directory that will be used to store all data and the configuration of the +service. + +Dump Things Service instances maintain a persistent configuration. The +configuration is stored in a private area of the *storage root* and will be +loaded when a Dump Things Service instance is started. + +The initial configuration can be provided in two ways: + +1. via a configuration file passed to the command line with the ``-c/--config`` + option. If not persistent configuration is found, a new Dump Things Service + instance will initialize the persistent configuration from the provided file. + +2. via the administration endpoints of a running service. To be able to use + the administration endpoint on a Dump Things Service instance with an empty + configuration, the command line option ``--admin-token-hash`` has to be + provided. This option will establish a "bootstrap" administrator token in the + Dump Things Server instance that can later be used to configure the service + via the administration endpoints (the tool ``dump-things-upload-config`` can + be used to load a configuration from a file and upload it to the running + service). Note: the bootstrap administrator token does not become part of + the persisted configuration. + + + +Elements of the configuration +----------------------------- + +Dump Things Service instances have three main configuration elements: + +- Collections +- Tokens +- Administrator Tokens + + +A Dump Things Service instance has administration endpoints for all three +configuration elements, which allow to create, modify, and delete +these elements. Any change to the configuration is persisted immediately and +will be effective for all subsequent requests to the service. + +All configuration changes are audited (see :doc:`audit` for details). The +audit log is stored in the storage root at the path +``/__dumpthings__/config_audit``. + + + +Configuration modification model +-------------------------------- + +A Dump Things Service instance maintains an internal representation of the +current configuration. The persistent configuration is kept up-to-date with +the internal configuration. + +If, for example, a collection is added via the administration +interface, the service will add the collection to its internal configuration +representation. It will then persist the new configuration and updated its +internal state to "activate" the new configuration. + +In other words if the administrator interface is used to add a collection, a +token, or an administrator-token, the element will be added to the already +existing configuration elements. If an element is removed via the administration +interface, this element will be removed from the internal configuration, while +all other configuration elements remain unchanged. + +To determine the current configuration of the service, the administrator API +provides endpoints to read collections, tokens, and admin tokens. + +To modify an existing configuration element, the administrator can fetch its +current state, modify it, and ``PUT`` it back to the service. + +The tool ``dump-things-download-config`` (see :doc:`commands`) can be used to +fetch the current configuration of a running Dump Things Service instance and +write it to a configuration file (see :ref:`config_file:`). This file can be +used, for example, to create a Dump Things Service with an identical +configuration at another location. + + +For a detailed description of the configuration endpoints see :doc:`endpoints`. + + +.. _config_file: Configuration File Structure ----------------------------- @@ -23,6 +107,10 @@ A configuration file (version 2) has the following top-level structure: : ... + admin + + + Collections ----------- diff --git a/docs/images/dts-author-write.svg b/docs/images/dts-author-write.svg new file mode 100644 index 0000000..b209407 --- /dev/null +++ b/docs/images/dts-author-write.svg @@ -0,0 +1,484 @@ + + + + +Created by potrace 1.16, written by Peter Selinger 2001-2019 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/dts-authors-reading.svg b/docs/images/dts-authors-reading.svg new file mode 100644 index 0000000..415433b --- /dev/null +++ b/docs/images/dts-authors-reading.svg @@ -0,0 +1,625 @@ + + + + +Created by potrace 1.16, written by Peter Selinger 2001-2019 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/dts-curate.svg b/docs/images/dts-curate.svg new file mode 100644 index 0000000..7dce9b0 --- /dev/null +++ b/docs/images/dts-curate.svg @@ -0,0 +1,658 @@ + + + + +Created by potrace 1.16, written by Peter Selinger 2001-2019 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/images/dts-reader.svg b/docs/images/dts-reader.svg new file mode 100644 index 0000000..7c1107a --- /dev/null +++ b/docs/images/dts-reader.svg @@ -0,0 +1,411 @@ + + + + +Created by potrace 1.16, written by Peter Selinger 2001-2019 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/index.rst b/docs/index.rst index 13d0ab1..8f92d6b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,7 +27,6 @@ acquisition. :maxdepth: 1 :caption: Developer Reference - api/index changelog Indices and tables diff --git a/docs/introduction.rst b/docs/introduction.rst index 20b3eae..63607c0 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -4,6 +4,208 @@ Introduction What is Dump Things Service? ----------------------------- +*Dump Things Service* is a central component of a system that allows cooperative +authoring of linked data (*linked data* refers to data that +is structured according to a schema which is defined in +`LinkML `_). It supports a workflow where authors submit records +into inboxes and curators move records from inboxes to a curated area. +The curated area holds the "official", high-quality, content. + +A curation workflow moves proposed changes from the inboxes into the +curated area. Curators inspect proposed changes, accept, modify, or reject them. +Curation can be performed manually or by automated agents. The service provides +tools and APIs to support both (all functionality is exposed via an HTTP API). + + + +The main elements +----------------- + +A Dump Things Service instance (usually a process that runs on a server) +supports multiple *Collections*. Each collection has a name and an associated +schema. All records that are stored in the collection must conform to a +schema that is associated with the collection. Every records has at least a +persistent identifier (PID) that is a globally unique identifier (more about +schemas below). + + +Clients that work with records on a Dump Things Service usually have one or +more of the following roles: + +- Readers: users that read data from the curated area of a collection. +- Authors: users that can submit new records for inclusion into the curated + area of a collection. Their submissions are stored in author-specific inboxes. + Authors can usually also read data from the collection. +- Curators: users that can curate the collection. Curators have read and write + access to the curated area of a collection. They also have read and write + access to all inboxes. Curators usually move records from inboxes to the + curated area, probably with editorial changes. + + +Besides an API that allows access to records, there is another API that +allows access to the configuration of a Dump Things Service and permits +dynamic configuration of an Dump Things Service instance at runtime. +Entities with an *Admin* role can use this API to change the configuration +of a Dump Things Service at runtime. + + +Interaction with Dump Things Service +------------------------------------ + +Depending on the role of a user, there are different interaction patterns with +a Dump Things Service. Those are: *Reading*, *Writing*, and *Curating*. + + +Reading +....... + +The most elementary interaction is to read records from +a collection. Depending on the role of a user, reading will either return only +records from the curated area or records from the curated area and from the +user's inbox. + +A user with read-access for a collection will be able to retrieve any record +that is stored in the curated area of a collection. + +.. image:: images/dts-reader.svg + :alt: Reader access to curated area + :align: center + +A user with read- and write-access for a collection, i.e., an author, will be +able to retrieve any record that is stored in the curated area of a collection +or in the inbox of the user. If records from the curated area and the inbox +have identical PIDs, the record from the inbox will be returned. + +.. image:: images/dts-authors-reading.svg + :alt: Reading access of authors to curated area and inbox + :align: center + + +Writing +....... + +A user with write-access for a collection, i.e., an author, can submit new +records to the collection. If the records confirm the schema of the collection, +they will be stored in the inbox of the user. Note: write-access does not allow +to move records from the inbox to the curated area. + +.. image:: images/dts-author-write.svg + :alt: Authors write to their "personal" inboxes + :align: center + + + +Curating +......... + +The purpose of curating is to moves proposed changes from the inboxes of a +collection to the curated area of the collection. + +Curator-rights allow a user to read, write, +and delete records from the curated area directly. The user can also read, +write, and delete records from any inbox of the collection. + +.. image:: images/dts-curate.svg + :alt: Curator access to curated area and inboxes + :align: center + + +Curators use the API to inspect proposed changes, accept, modify, or reject +them. The client library `dump-things-pyclient` (see :ref:`dump-things-pyclient`) +provides tools that support curation processes. + + +Administration +.............. + +Administrators do not operate on records. Administrators can inspect and modify +the configuration of a Dump Things Service instance at runtime. An administrator +can modify three configuration elements: + +1. Collections: add, remove, or modify collections. +2. Tokens: add, remove, or modify tokens. +3. Administrator token: add. remove, or modify administrator tokens. + + + + +.. _dump-things-pyclient: +The dump-things ecosystem +------------------------- + +Dump Things Service is part of a larger ecosystem of tools and services that +support the shared authoring and curation of linked data. Currently there are +two other components that are built to work with it: + +- shacl-vue: a web application that allows users to view and edit linked data + records stored in a Dump Things Service instance. + +- dump-things-pyclient: a Python client library that simplifies authoring of + Dump Things clients in Python. It also provides a CLI-command to interact + with a Dump Things Service from a shell. + + + +Dump Things Structure +--------------------- + +Workflow +........ + +Dump Things Service was built to support cooperative authoring of linked data. +It supports a workflow where authors submit records into inboxes and curators +move records from inboxes to a curated area. + + +Dump Things Service supports distributed data acquisition while still +controlling the quality of collection data. This is done by distinguishing +between two areas for records: + +- **Curated area**: represents the "official", high-quality, collection content. +- **Incoming area** (user-specific inboxes): where users propose new records or + changes to existing records. + +A curation workflow moves proposed changes from the incoming area into the +curated area. Curators — entities with specific privileges — can inspect +proposed changes, accept, modify, or reject them. Curation can be performed by +persons or by automated agents. The service provides tools and APIs to support +both options. + + + + + +Collections +........... + +Dump Things Service supports on the highest level multiple *collections*. Each +collection has a name and an associated schema. Collections have inboxes for +all authors and a curated area. + + + + +User groups +........... + + +We distinguish he following user groups: + +- Readers: user that read data from a collection. +- Authors: users that can submit new records to a collection. Authors can + usually also read data from the collection. +- Curators: users that can curate the collection. Curators have read and write + access to the curated area of a collection. They also have read and write + access to all inboxes. Curators usually move records from inboxes to the + curated area, probably with editorial changes. + + + + + + + + *Dump Things Service* is an implementation of a service that allows storing and retrieving linked data. It builds the backend for linked-data applications. Linked-data applications can retrieve data as JSON records or as @@ -58,5 +260,7 @@ Key Features `Forgejo `_-based authentication. - **Audit logs**: track time, content, author, and curator for every change. - **LinkML-based validation**: ensure data conforms to the defined schema. + - **GraphQL support**: query data via a GraphQL interface. + - **Pagination**: all list endpoints support optional pagination. -- 2.52.0 From 0cb97fd018e8a6db457b91a5e474e3922485b665 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 14 Jul 2026 12:12:18 +0200 Subject: [PATCH 4/8] doc: update quickstart and installation docs --- docs/installation.rst | 4 +- docs/quickstart.rst | 106 ++++++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 41 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 78af667..1104c5a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -5,7 +5,7 @@ Requirements ------------ - Python 3.11 or later -- ``sqlite3`` (usually bundled with Python) +- ``sqlite3`` (if sqlite-backends are used) Installing from PyPI --------------------- @@ -40,7 +40,7 @@ group. To install them and build the HTML documentation: pip install "dump-things-service[docs]" make -C docs html -Alternatively, if you are using ``hatch``: +Alternatively, if you are using ``hatch`` execute the following command from the project root: .. code-block:: bash diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 52865a7..4e810b0 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,67 +1,79 @@ Quick Start =========== -This guide walks you through starting the service with a minimal configuration. +This guide walks you through starting the service with or without an initial +configuration. + +The service is started with the command ``dump-things-service ``. +```` is a directory that will be used to store records for all +collections as well as the persisted configuration of the service. + 1. Start the Service --------------------- -After installation, start the service with: +On an empty storage root +........................ + +If the directory ```` does not contain a persisted configuration, +for example, because it is empty, the service will start with an empty configuration. +In this case a *bootstrap* administrator token has to be provided via the option +``--admin-token-hash`` (otherwise it +would not be possible to modify the configuration are runtime because no +administrator token has been configured). + +The following command starts the server on ``https://127.0.0.1:8000/`` with the +bootstrap administrator token ``mysecret``: .. code-block:: bash - dump-things-service /path/to/storage --host 127.0.0.1 --port 8000 + dump-things-service /path/to/storage --host 127.0.0.1 --port 8000 --admin-token-hash 652c7dc687d98c9889304ed2e408c74b611e86a40caa51c4b43f1dd5913c5cd0 -Where ``/path/to/storage`` is a directory that will be used as the storage root. -Unless ``-c/--config`` is provided, the service looks for a configuration file -at ``/.dumpthings.yaml``. -Alternatively, supply a configuration file explicitly: +**Note**: an administrator token hash can be created via the command +``dump-things-hash-token ``. -.. code-block:: bash +**Note**: do not use the administrator token hash from the example above in your +installations, as it is publicly visible. - dump-things-service /path/to/storage \ - --host 127.0.0.1 --port 8000 \ - -c /path/to/config.yaml -2. Set an Administrator Token ------------------------------- - -An administrator token is required to create collections and tokens. -It can be provided as a SHA-256 hash on the command line: - -.. code-block:: bash - - # Generate a hash for the plain-text token "mysecret" - dump-things-hash-token 'mysecret' - # → <64-character hex string> - - # Start the service with the hashed admin token - dump-things-service /path/to/storage \ - --admin-token-hash <64-character hex string> - -Alternatively, set the environment variable ``DTS_ADMIN_TOKEN`` to the -plain-text token before starting the service: +Alternatively to setting ``--admin-token-hash``, set the environment variable +``DTS_ADMIN_TOKEN`` to the plain-text bootstrap administrator token, before starting the service: .. code-block:: bash export DTS_ADMIN_TOKEN=mysecret dump-things-service /path/to/storage -3. Upload a Configuration --------------------------- -The easiest way to configure collections and tokens is to write a YAML -configuration file (see :doc:`configuration`) and upload it with: +**Note**: do not use ``mysecret`` in your installations as it is publicly visible. +Use a strong secret instead. -.. code-block:: bash - dump-things-upload-config \ - --server http://127.0.0.1:8000 \ - --admin-token mysecret \ - /path/to/config.yaml +On an empty storage root, initially configured via config-file +.............................................................. -4. Explore the API +If the storage root does not contain a persisted configuration, the service will +look for a configuration file at ``/.dumpthings.yaml``, unless +``--ignore-default-config-file`` or +``-c/--config `` is provided. + +If ``--ignore-default-config`` is +provided, the server will ignore a configuration file at +``/.dumpthings.yaml``. + +If ``-c/--config `` is passed +on the command line, the server will +try to read the configuration from ````. Once the configuration is +read, it will be persisted in the storage root. + +See :doc:`configuration` for details on the configuration file format. + +**Note**: on subsequent restarts, the service will ignore the configuration files +and use the persisted configuration. + + +2. Explore the API ------------------ Once the service is running, open your browser at @@ -69,3 +81,19 @@ Once the service is running, open your browser at available endpoints. For a full description of all endpoints see :doc:`endpoints`. + + +3. Configure the service +------------------------ + +The service configuration can be modified at runtime via the administration +endpoints. See :doc:`endpoints` for details. + +An easy way to configure a running service is to use the command +``dump-things-upload-configuration`` to upload the content of a +configuration file via the administration endpoints (see :doc:`configuration`). + +In combination with the command ``dump-things-download-config`` this command +can be used to "clone" an identically configured service instance (the cloned +service instance will have no records initially, but those can be cloned +from the original service instance as well). -- 2.52.0 From ef824d07b4f3ae964ee7b667d46c1bbd67ffe392 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 15 Jul 2026 10:51:47 +0200 Subject: [PATCH 5/8] docs: update documentation Update the documentation parts: - introduction - configuration - authentication - commands - backends --- docs/authentication.rst | 217 ++++++++++++------------------ docs/backends.rst | 38 +----- docs/commands.rst | 136 ++++++++----------- docs/configuration.rst | 285 +++++++++++++++++++++++++--------------- docs/introduction.rst | 17 ++- 5 files changed, 348 insertions(+), 345 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 90abbb4..b87d137 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -10,7 +10,7 @@ Authentication Sources Authentication sources are tried in the order they are listed for a collection. If an authentication source authenticates the token successfully, no further sources are queried. If no source is defined for a collection, the -``config``-source is used. +``config``-source is used, i.e., tokens are read directly from the configuration. If the same authentication source is listed more than once, the duplicate entries are ignored and a warning ``Ignoring duplicate authentication provider...`` is @@ -19,14 +19,17 @@ logged. Available Sources ^^^^^^^^^^^^^^^^^ +The following authentication source types are available: + ``config`` - Reads token information directly from the configuration file. + Reads token information directly from the persisted configuration. This is the default source. ``forgejo`` Uses a `Forgejo `_ instance to authenticate tokens and to determine the user ID and the incoming label. + Configuring Authentication Sources ------------------------------------ @@ -48,6 +51,7 @@ key: repository: reference-repository # optional - type: config + Config-based Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -58,151 +62,104 @@ No extra keys are required beyond ``type: config``. auth_sources: - type: config + Forgejo-based Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The Forgejo source uses the Forgejo REST API to validate tokens and determine -permissions. The token must have at least these Forgejo API permissions: +permissions. This allows clients to use Forgejo personal access tokens to +authenticate with the service. -- **User** (Read): required to get user email and login name. -- **Organization** (Read): required to determine team membership. -- **Repository** (Read): required only when ``repository`` is set. +This is an example configuration for a Forgejo source: -.. list-table:: Forgejo source configuration keys - :header-rows: 1 - :widths: 25 10 65 +.. code-block:: yaml + + collections: + my_collection: + auth_sources: + - type: forgejo + url: https://forgejo.example.com/api/v1 + organization: data_handling + team: data_entry_personal + label_type: team + repository: reference-repository # optional + instance_id: example_com + +The keys are: + +``type`` + Must be ``forgejo``. + +``url`` + The API URL of the Forgejo instance, e.g. ``https://forge.example.com/api/v1``. + +``organization`` + The name of the Forgejo organization that is used to determine the permissions of the token. + +``team`` + The name of the team inside the organization that is used to determine the permissions of the token. + +``label_type`` + Either ``team`` or ``user``. Determines the incoming label format. + + If `label_type` is set to `team`, the incoming label is `forgejo-team--`. + + If `label_type` is set to `user`, the incoming label is `forgejo-user-` + + +``repository`` (optional) + If set, the token is only authorized if the team has access to this + repository. + +``instance_id`` (optional) + Optional identifier to disambiguate users on different Forgejo instances. + Defaults to a hash of ``url``. + + +When a Forgejo source authenticates a token, the user ID is set to the email of the Forgejo user. + +The permissions will be fetched from the units `repo.code` and `repo.actions` of the team definition. +The following mapping is used: + +============ ============ ============= ============== ============= ============ +`repo.code`: curated_read incoming_read incoming_write curated_write zones_access +============ ============ ============= ============== ============= ============ +`none` `False` `False` `False` `False` `False` +`read` `True` `True` `False` `False` `False` +`write` `True` `True` `True` `False` `False` +============ ============ ============= ============== ============= ============ + + +=============== ============ ============= ============== ============= ============ +`repo.actions`: curated_read incoming_read incoming_write curated_write zones_access +=============== ============ ============= ============== ============= ============ +`none` `False` `False` `False` `False` `False` +`read` `False` `False` `False` `False` `False` +`write` `True` `True` `True` `True` `True` +=============== ============ ============= ============== ============= ============ + + + +A Forgejo authentication source can authenticate Forgejo-tokens that have at +least the following `Read`-permissions: + +- User: this is required to determine user-related information, i.e. user-email and user login name. +- Organization: this is required to determine the membership of a user to a team in an organization. +- (Only if `repository` is set in the configuration) Repository: required to determine a team's access to the repository. - * - Key - - Required - - Description - * - ``url`` - - ✓ - - API URL of the Forgejo instance, e.g. ``https://forge.example.com/api/v1``. - * - ``organization`` - - ✓ - - Name of the Forgejo organization. - * - ``team`` - - ✓ - - Name of the team inside the organization. - * - ``label_type`` - - ✓ - - ``team`` or ``user``. Determines the incoming label format (see below). - * - ``repository`` - - ✗ - - If set, the token is only authorized if the team has access to this repository. - * - ``instance_id`` - - ✗ - - Optional identifier to disambiguate users on different Forgejo instances. Defaults to a hash of ``url``. Incoming Label Generation '''''''''''''''''''''''''' The ``label_type`` key controls how the incoming label is derived: -- ``team`` → ``forgejo-team--`` -- ``user`` → ``forgejo-user-`` +- ``team`` → ``forgejo--team--`` +- ``user`` → ``forgejo--user-`` -Permission Mapping -'''''''''''''''''' -Forgejo team permissions are mapped to service permissions as follows: -.. list-table:: Mapping of ``repo.code`` to service permissions - :header-rows: 1 - - * - ``repo.code`` - - curated_read - - incoming_read - - incoming_write - - curated_right - - zones_access - * - ``none`` - - ✗ - - ✗ - - ✗ - - ✗ - - ✗ - * - ``read`` - - ✓ - - ✓ - - ✗ - - ✗ - - ✗ - * - ``write`` - - ✓ - - ✓ - - ✓ - - ✗ - - ✗ - -.. list-table:: Mapping of ``repo.actions`` to service permissions - :header-rows: 1 - - * - ``repo.actions`` - - curated_read - - incoming_read - - incoming_write - - curated_right - - zones_access - * - ``none`` - - ✗ - - ✗ - - ✗ - - ✗ - - ✗ - * - ``read`` - - ✗ - - ✗ - - ✗ - - ✗ - - ✗ - * - ``write`` - - ✓ - - ✓ - - ✓ - - ✓ - - ✓ - -Access Modes -------------- - -Token permissions for a collection are expressed as a *mode*: - -.. list-table:: - :header-rows: 1 - :widths: 30 70 - - * - Mode - - Granted permissions - * - ``NOTHING`` - - No access at all. - * - ``READ_CURATED`` - - Read curated data only. - * - ``READ_INCOMING`` - - Read own incoming zone. - * - ``WRITE_INCOMING`` - - Read & write own incoming zone; read curated. - * - ``WRITE_COLLECTION`` - - Full access: read curated + read/write own incoming zone. - * - ``CURATOR`` - - Full access plus read/write all incoming zones and curated. Administrator Tokens --------------------- -Administrator tokens are required for the administration endpoints (creating / -deleting collections, tokens, etc.). They are **always** stored as SHA-256 -hashes. To create one: - -.. code-block:: bash - - # Compute the hash - dump-things-hash-token 'my-admin-secret' - - # Pass the hash when starting the service - dump-things-service /path/to/storage \ - --admin-token-hash <64-digit hex hash> - -Alternatively, set ``DTS_ADMIN_TOKEN`` to the plain-text token in the -environment before starting the service. The command-line option takes -precedence if both are provided. +Administrator tokens are always authenticated using the persisted configuration. diff --git a/docs/backends.rst b/docs/backends.rst index 5934541..b3ea1b8 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -29,6 +29,8 @@ Available Backends ``sqlite`` Stores records in a SQLite database. One database file named ``__sqlite-records.db`` is created per curated/incoming area. + ``sqlite``-backends have the advantage that they support search patterns + to limit the size of a result on the server. ``sqlite+stl`` Same as ``sqlite``, but with the same schema-type layer as @@ -36,9 +38,10 @@ Available Backends .. note:: - All backends can co-exist independently in the same directory. There are no - file-name collisions between backends, as long as no class name starts with - ``.`` or ``_``. + ``record_dir`` and ``sqlite`` backends can co-exist independently in the same + directory. There are no file-name collisions between backends, as long as no + class name starts with ``.`` or ``__``. + Configuring a Backend --------------------- @@ -66,32 +69,3 @@ Specify the backend in the collection configuration with the ``backend`` key: backend: type: sqlite - -Migrating Between Backends ---------------------------- - -Use ``dump-things-copy-store`` to copy data between backends -(see :doc:`commands`). - -Example — ``record_dir`` → ``sqlite``: - -.. code-block:: bash - - dump-things-copy-store \ - record_dir:/path/to/data/curated \ - sqlite:/path/to/data/curated - -Example — ``record_dir+stl`` → ``sqlite`` (schema required): - -.. code-block:: bash - - dump-things-copy-store \ - --schema https://concepts.inm7.de/s/flat-data/unreleased.yaml \ - record_dir+stl:/path/to/data/curated \ - sqlite:/path/to/data/curated - -.. note:: - - When copying from a ``record_dir`` or ``record_dir+stl`` backend that was - modified outside the service (e.g. by manually adding/removing files), run - ``dump-things-rebuild-index`` first to ensure the index is up to date. diff --git a/docs/commands.rst b/docs/commands.rst index 9362ad0..fe721d2 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -19,8 +19,7 @@ Start the Dump Things Service. ```` (Mandatory) Path to the directory used as anchor for all relative paths in - the configuration. Unless ``-c/--config`` is provided, the service looks for - a configuration file at ``/.dumpthings.yaml``. + the configuration. **Options** @@ -37,7 +36,7 @@ Start the Dump Things Service. Path to a configuration file. If no persisted configuration is found in the data store, the service is initialized with the content of this file. - Useful for migrating from service version 5 and lower to version 6. + Useful for migrating from service version <= 5 to version 6. .. option:: --origins @@ -55,13 +54,17 @@ Start the Dump Things Service. .. option:: --admin-token-hash Set an administrator token hash (64-digit hexadecimal SHA-256 hash of the - plain-text token). If not provided, the plain-text token is read from the - environment variable ``DTS_ADMIN_TOKEN``. + plain-text token). If not provided, a plain-text token is read from the + environment variable ``DTS_ADMIN_TOKEN``, if it is set. .. option:: --ignore-default-config-file If set, the service will not try to initialize itself from the default - configuration file ``/.dumpthings.yaml``. + configuration file ``/.dumpthings.yaml``. Otherwise, if no + persisted configuration is found in the data store (and ``-c/--config`` is + not provided), the service will read its initial configuration from + ``/.dumpthings.yaml`` if it exists. + **Example** @@ -69,18 +72,23 @@ Start the Dump Things Service. dump-things-service /data/store --host 127.0.0.1 --port 8000 + Configuration Tools ------------------- dump-things-upload-config ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Read a YAML configuration file and apply it to a running service via the -administration endpoints. +Read a configuration from a dump-things configuration-file and +instantiate its elements on a running server via the administration +enpoints. Objects that already exist on the server are left unchanged. + +An admin token has to be provided in the environment variable +``DTS_ADMIN_TOKEN``. .. code-block:: text - dump-things-upload-config [OPTIONS] + dump-things-upload-config [OPTIONS] **Options** @@ -88,73 +96,61 @@ administration endpoints. Base URL of the running service (e.g. ``http://127.0.0.1:8000``). -.. option:: --admin-token +.. option:: --format [{json,yaml}], -f [{json,yaml}] + + Specify the format of the input file. Possible values are `json` and `yaml`. If this option is given, the suffix of the configuration file is ignored. + +.. option:: --send-to SEND_TO + + The base URL of the server API. If this option is provided, the configuration will be sent to the server API, otherwise it will just be written to stdout. + +.. option:: --old-format + + If provided, assume that the configuration is in version 1 format and convert it to the new format internally (in version 1: tokens had no `hashed`-attribute and no `representation`-attribute, the token representation was the key of the token configuration, collections had no `schema`-attribute, and `sqlite`-backends had a `schema`-attribute). + +.. option:: --store STORE + + If --old-format is provided, this option can be used to specify a store directory. The store directory will be used to load `RecordDir` configurations, if a collection defines are `RecordDir`-backend. (This option has no effect if no collection in the old configuration uses a `RecordDir`-backend.) - Plain-text administrator token. dump-things-download-config ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Read the current configuration from a running service and write it to a file. +An admin token has to be provided in the environment variable +``DTS_ADMIN_TOKEN``. + .. code-block:: text - dump-things-download-config [OPTIONS] + dump-things-download-config [OPTIONS] **Options** -.. option:: --server +.. option:: --entities {admin_tokens,collections,tokens}, -e {admin_tokens,collections,tokens} + + Specify for which entities the configuration should be downloaded. Possible values are `admin_tokens`, `collections`, or `tokens` (repeat to download configuration for more than one entity). If this option is not provided, configurations for all entities will be downloaded. + +.. option:: --format [{json,yaml}], -f [{json,yaml}] + + Specify the format of the output. Possible values are `json` and `yaml` (the default is `yaml`). + + +dump-things-hash-token +^^^^^^^^^^^^^^^^^^^^^^^^ + +Compute the SHA-256 hash of a plain-text token for use with ``--admin-token-hash`` or as ``representation``-value in a configuration object (file) + +.. code-block:: bash + + dump-things-hash-token - Base URL of the running service. -.. option:: --admin-token - Plain-text administrator token. Maintenance Tools ----------------- -dump-things-rebuild-index -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Rebuild the persistent index of a ``record_dir`` store. Run this after the -store was modified outside the service (e.g. by manually adding/removing -files). - -.. code-block:: bash - - dump-things-rebuild-index - -dump-things-copy-store -^^^^^^^^^^^^^^^^^^^^^^^ - -Copy a collection from a source store to a destination store. The source and -destination are specified as ``:`` where ```` is one of -``record_dir``, ``record_dir+stl``, ``sqlite``, or ``sqlite+stl``. - -.. code-block:: bash - - dump-things-copy-store [--schema ] - -.. option:: -s, --schema - - Required when either backend is ``record_dir+stl`` or ``sqlite+stl``. - -**Examples** - -.. code-block:: bash - - # record_dir → sqlite - dump-things-copy-store \ - record_dir:/path/to/curated \ - sqlite:/path/to/curated - - # record_dir+stl → sqlite (schema required) - dump-things-copy-store \ - --schema https://concepts.inm7.de/s/flat-data/unreleased.yaml \ - record_dir+stl:/path/to/curated \ - sqlite:/path/to/curated - dump-things-pid-check ^^^^^^^^^^^^^^^^^^^^^^ @@ -165,25 +161,7 @@ schema changes that affect prefix definitions). dump-things-pid-check -dump-things-create-merged-schema -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Create a fully self-contained schema that statically embeds all schemas -imported by the original schema. - -.. code-block:: bash - - dump-things-create-merged-schema - -dump-things-hash-token -^^^^^^^^^^^^^^^^^^^^^^^^ - -Compute the SHA-256 hash of a plain-text token for use with -``--admin-token-hash``. - -.. code-block:: bash - - dump-things-hash-token Audit Tools ----------- @@ -191,14 +169,15 @@ Audit Tools dump-things-gitaudit-report ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Show the audit log for all PIDs matching a pattern. +Report the audit information that was stored for all PIDs matching a pattern. +For every change to a record the tool will report: time stamp, user ID, diff, and the resulting record. .. code-block:: bash dump-things-gitaudit-report -```` is a Python ``re``-module pattern. -Use ``'.*'`` to report changes for all PIDs. +```` is a Python ``re``-module pattern that identifies PIDs of the record for +which audit information should be reported. Use ``'.*'`` to report changes for all PIDs. Each log entry contains: @@ -207,6 +186,7 @@ Each log entry contains: - Diff of the change. - The resulting record. + dump-things-gitaudit-rebuild-index ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/configuration.rst b/docs/configuration.rst index ed55499..79898a1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,31 +5,41 @@ Configuration General Concepts ---------------- -A Dump Things Service instance is created by executing the command -``dump-things-service`` on the command line. The command has one mandatory -argument, the path to the **storage root**. This is a path to a locally accessible -directory that will be used to store all data and the configuration of the -service. - -Dump Things Service instances maintain a persistent configuration. The +Each Dump Things Service instance maintains a persistent configuration. The configuration is stored in a private area of the *storage root* and will be loaded when a Dump Things Service instance is started. -The initial configuration can be provided in two ways: -1. via a configuration file passed to the command line with the ``-c/--config`` - option. If not persistent configuration is found, a new Dump Things Service - instance will initialize the persistent configuration from the provided file. +Providing an initial configuration +.................................. +If the persistent configuration is empty, for example, because the service is +started on an empty ````, an initial configuration can be +provided by configuration files: -2. via the administration endpoints of a running service. To be able to use - the administration endpoint on a Dump Things Service instance with an empty - configuration, the command line option ``--admin-token-hash`` has to be - provided. This option will establish a "bootstrap" administrator token in the - Dump Things Server instance that can later be used to configure the service - via the administration endpoints (the tool ``dump-things-upload-config`` can - be used to load a configuration from a file and upload it to the running - service). Note: the bootstrap administrator token does not become part of - the persisted configuration. +1. via a configuration file in ``/.dumpthings.yaml``, or + +2. a configuration file can be passed to the command line with the ``-c/--config`` + option. + + + +Performing dynamic configuration changes +........................................ + +Every Dump Things Service instance (since version 6) supports dynamic +reconfiguration via administration API endpoints. That means, a server can be +started with an empty configuration and the configuration can be modified at +runtime. + +If a Dump Things Service instance is started with an empty configuration, +the command line option ``--admin-token-hash`` has to be provided. +This option will establish a "bootstrap" administrator token in the +Dump Things Server instance that can be used to access the +administration endpoints. + +Note: the bootstrap administrator token does not become part of the persisted +configuration. That means, it will not be represented in a configuration dump, +i.e., in the result of ``GET /admin_tokens``. @@ -73,7 +83,8 @@ interface, this element will be removed from the internal configuration, while all other configuration elements remain unchanged. To determine the current configuration of the service, the administrator API -provides endpoints to read collections, tokens, and admin tokens. +provides endpoints to read collection configurations, token configurations, and +admin token configurations. To modify an existing configuration element, the administrator can fetch its current state, modify it, and ``PUT`` it back to the service. @@ -107,15 +118,17 @@ A configuration file (version 2) has the following top-level structure: : ... - admin + admin_tokens: + : + ... Collections ----------- -Each entry under ``collections`` defines a collection. The following keys are -supported: +Each entry under ```` defines a collection. The following keys +are supported: ``schema`` (required) URL of the `LinkML `_ schema for this collection. @@ -124,33 +137,46 @@ supported: Name of the token to use when no token is provided by a client. All permissions in the default token are OR-ed with any client-provided token. -``curated`` (required) - Path to the curated area of the collection. A relative path is interpreted - relative to the storage root. + Note: a token of the given name must be defined in the ``tokens`` section + of the configuration file. At least the default token must be created before + a collection can be created. Although tokens reference collections, tokens + can always be created because their creation does check whether referenced + collections exist yet. -``incoming`` (optional) - Path to the incoming area. Required if the collection should accept uploads. +``curated`` (required) + Path to the curated area of the collection. This should be a relative path, + which will be interpreted relative to the storage root. + +``incoming`` (required if tokens with write access exist for the collection) + Path to the incoming area. Required if the collection has any tokens with + write access associated to it. + This should be a relative path, which will be interpreted relative to the + storage root. ``backend`` (optional) Storage backend configuration. If omitted, ``record_dir+stl`` is used. See :doc:`backends` for details. ``use_classes`` (optional) - List of LinkML class names that should receive store/validate endpoints. - All other classes defined in the schema are ignored. + List of LinkML class names that should receive store/validate endpoints + (the classes must be subclasses of ``Thing``). All other classes defined + in the schema are ignored. ``ignore_classes`` (optional) - List of LinkML class names to exclude from store/validate endpoints. + List of LinkML class names to exclude from store/validate endpoints + (the classes must be subclasses of ``Thing``). ``auth_sources`` (optional) List of authentication source configurations. See :doc:`authentication`. ``submission_tags`` (optional) - Override the default annotation tags. See `Submission Annotation Tags`_ below. + Tags for an optional automated annotation of submitted records (see + `Submission Annotation Tags`_ below). ``audit_backends`` (optional) List of audit-backend configurations. See `Audit Backends`_ below. + Example ^^^^^^^ @@ -159,14 +185,104 @@ Example collections: personal_records: default_token: no_access - schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml - curated: curated/personal_records - incoming: /tmp/personal_records/incoming + schema: https://concepts.datalad.org/s/demo-research-information/unreleased.yaml + curated: curated + incoming: incoming use_classes: - Organization - Person - Project + +Authentication Sources +^^^^^^^^^^^^^^^^^^^^^^ + +Authentication configuration is described in :doc:`authentication`. If no +authentication sources are defined for a collection, a ``config`` source is +used by default, that means, the service will use the tokens that are +configured in the configuration file. + + + +Submission Annotation Tags +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The service might annotate uploaded records with a submitter ID and a +submission time (version below 6 will always annotate submitted records, +version 6 and higher will only annotate submitted records, if the query +parameter ``add_submission_tag`` is set to ``true``. + +By default, the following tags are used for submission annotation: + +- Submitter ID: ``http://purl.obolibrary.org/obo/NCIT_C54269`` +- Submission time: ``http://semanticscience.org/resource/SIO_001083`` + +Override these defaults per collection with ``submission_tags``: + +.. code-block:: yaml + + collections: + collection_1: + ... + submission_tags: + submitter_id_tag: schema:user_id + submission_time_tag: schema:time + + +Audit Backends +^^^^^^^^^^^^^^ + +The service supports audit logs of changes made via the curation interface. +Configure them per collection with ``audit_backends``: + +.. code-block:: yaml + + collections: + collection_1: + ... + audit_backends: + - type: gitaudit + path: /path/to/audit-log-dir + auto_flush_timeout: 60 # seconds (default: 60, minimum: 1) + +Currently the only supported audit backend type is ``gitaudit``. A ``gitaudit`` +backend stores changes in a bare Git repository at ``path``. +After ``auto_flush_timeout`` seconds without new entries the current changeset +is committed. + + +Use ``dump-things-gitaudit-report`` to display the audit log and +``dump-things-gitaudit-rebuild-index`` to rebuild its index (see :doc:`commands`). + + + +Collection-and-Token: a Chicken-and-Egg Problem +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In configuration files, collections reference tokens and tokens reference collections. +This can create a chicken-and-egg problem when creating new collection and +new tokens dynamically. This is resolved by allowing tokens to refer to +collections that are not created. In other words: + + Create tokens first, then create collections. + + + + + +Reserved Collection Names +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following collection names are reserved and must **not** be used: + +- ``api`` +- ``collections`` +- ``tokens`` +- ``admin_tokens`` +- ``__dump_things__`` + + + Tokens ------ @@ -179,7 +295,7 @@ Each entry under ``tokens`` defines a token. The following keys are supported: ``hashed`` (optional, default: ``False``) If ``True``, the representation is treated as a SHA-256 hash. - Use ``dump-things-hash-token`` to compute the correct hash. + Use the command ``dump-things-hash-token`` (see :doc:commands) to compute the correct hash. ``user_id`` (required) An identifier that is added as an annotation to each uploaded record. @@ -187,24 +303,31 @@ Each entry under ``tokens`` defines a token. The following keys are supported: ``collections`` (required) Mapping from collection name to per-collection rights. + Per-collection rights ^^^^^^^^^^^^^^^^^^^^^ Each collection entry inside a token has: ``mode`` - Access mode. Available modes: + Access mode. Available modes (in the following list "own incoming area" + refers to the incoming area that is associated with the token and the + collection): - ``NOTHING`` – no access. - - ``READ_CURATED`` – read curated data only. - - ``READ_INCOMING`` – read own incoming zone. - - ``WRITE_INCOMING`` – read & write own incoming zone, read curated. - - ``WRITE_COLLECTION`` – full access (read curated + read/write incoming). - - ``CURATOR`` – curator-level access (curated read/write + all incoming). + - ``READ_CURATED`` – read the curated area only. + - ``READ_COLLECTION`` – read curated data and own incoming area. + - ``WRITE_COLLECTION`` – read curated data, read own incoming area, write to own incoming area. + - ``READ_SUBMISSIONS`` - read own incoming area. + - ``WRITE_SUBMISSIONS`` – read and write own incoming area. + - ``SUBMIT`` - read curated area only, write to own incoming area. + - ``SUBMIT_ONLY`` - write to own incoming area. + - ``CURATOR`` - curator-level access: read & write to curated area, read & write to all incoming areas. + ``incoming_label`` - Label that defines the zone in the incoming area for this token/collection. - Multiple tokens can share the same label, which means they share the zone. + Label that defines the subdirectory (aka zone) in the incoming area for this token and collection. + Multiple tokens can share the same label, which means they share a zone. Example ^^^^^^^ @@ -229,57 +352,6 @@ Example mode: WRITE_COLLECTION incoming_label: new_rooms_and_buildings -Reserved Collection Names --------------------------- - -The following collection names are reserved and must **not** be used: - -- ``collections`` -- ``tokens`` -- ``admin_tokens`` -- ``__dump_things__`` - -Submission Annotation Tags ---------------------------- - -By default the service annotates uploaded records with: - -- Submitter ID: ``http://purl.obolibrary.org/obo/NCIT_C54269`` -- Submission time: ``http://semanticscience.org/resource/SIO_001083`` - -Override these defaults per collection with ``submission_tags``: - -.. code-block:: yaml - - collections: - collection_1: - ... - submission_tags: - submitter_id_tag: schema:user_id - submission_time_tag: schema:time - -Audit Backends --------------- - -The service supports audit logs of changes made via the curation interface. -Configure them per collection with ``audit_backends``: - -.. code-block:: yaml - - collections: - collection_1: - ... - audit_backends: - - type: gitaudit - path: /path/to/audit-log-dir - auto_flush_timeout: 60 # seconds (default: 60, minimum: 1) - -The ``gitaudit`` backend stores changes in a bare Git repository at ``path``. -After ``auto_flush_timeout`` seconds without new entries the current changeset -is committed. - -Use ``dump-things-gitaudit-report`` to display the audit log and -``dump-things-gitaudit-rebuild-index`` to rebuild its index (see :doc:`commands`). Tips & Tricks ------------- @@ -307,16 +379,21 @@ path: mode: WRITE_COLLECTION incoming_label: curated # datamgt/curated == curated path -Migrating from Version 5 (or Lower) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Use the ``-c/--config`` option on first start to initialise the persisted -configuration from an existing file. Once started, the configuration is stored -in the internal database and the ``-c`` option can be omitted on subsequent -starts: +Migrating to Version 6 from Version <=5 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start the version 6 server on the same storage root as the version 5 server. +The version 6 server will automatically migrate the configuration from +``/.dumpthings.yaml`` from the version 5 format to the new +format and persist it. Use the ``-c/--config`` option on first start to +initialise the persisted configuration from a different file than +``/.dumpthings.yaml``. + +Once started, the configuration is stored in the internal database and the +file ``/.dumpthings.yaml`` or the ``-c`` option can be omitted +on subsequent starts: .. code-block:: bash dump-things-service /path/to/storage -c /path/to/old-config.yaml - -See also the migration guide in the :doc:`changelog`. diff --git a/docs/introduction.rst b/docs/introduction.rst index 63607c0..95b0d21 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -17,7 +17,6 @@ Curation can be performed manually or by automated agents. The service provides tools and APIs to support both (all functionality is exposed via an HTTP API). - The main elements ----------------- @@ -128,6 +127,22 @@ can modify three configuration elements: +Records and Schemas +------------------- + +Records that are stored in a collection are instances of the classes that are +defined in the schema of the collection. The schema can be freely defined, +but all record-classes, i.e., classes whose instances are can be stored in a +collection, must be a subclass of the class ``Thing``, which is +defined in the schema `things-schema`_. The crucial property of the class +``Thing`` is the persistent identifier (PID). The PID is an **IRI** +(Internationalized Resource Identifier), a globally unique +identifier that identifies the record. + + +.. _things-schema: https://concepts.datalad.org/s/things/v2 + + .. _dump-things-pyclient: The dump-things ecosystem -- 2.52.0 From f9bada4ec08fe588e39bfbb882dcbde95d3b0b0b Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 16 Jul 2026 09:01:53 +0200 Subject: [PATCH 6/8] doc: update documentation update: docs/autenticate.rst docs/endpoints.rst docs/introduction.rst --- docs/authentication.rst | 6 ++-- docs/endpoints.rst | 79 +++++++++++++++++++++++++++++++++++++---- docs/introduction.rst | 6 +--- 3 files changed, 76 insertions(+), 15 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index b87d137..8a5cf5f 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -143,9 +143,9 @@ The following mapping is used: A Forgejo authentication source can authenticate Forgejo-tokens that have at least the following `Read`-permissions: -- User: this is required to determine user-related information, i.e. user-email and user login name. -- Organization: this is required to determine the membership of a user to a team in an organization. -- (Only if `repository` is set in the configuration) Repository: required to determine a team's access to the repository. +- **User**: this is required to determine user-related information, i.e. user-email and user login name. +- **Organization**: this is required to determine the membership of a user to a team in an organization. +- **Repository** (Only if `repository` is set in the configuration): required to determine a team's access to the repository. Incoming Label Generation diff --git a/docs/endpoints.rst b/docs/endpoints.rst index 5bee715..d53ff32 100644 --- a/docs/endpoints.rst +++ b/docs/endpoints.rst @@ -11,6 +11,14 @@ token supplied in the ``X-DumpThings-Token`` HTTP header. :local: :depth: 1 + +Note: The package `dump-things-pyclient`_ provides client support for +the service. A Python API makes the endpoints available to Python code. The +command line tool ``dtc`` makes the endpoints available in the shell. + + +.. _dump-things-pyclient: https://pypi.org/project/dump-things-pyclient/ + User Endpoints -------------- @@ -161,17 +169,43 @@ Curation Endpoints ------------------- Curation endpoints allow direct read/write access to the curated area and all -incoming zones. A ``CURATOR`` token is required. +incoming zones (aka: inboxes). A ``CURATOR`` token is required. Refer to the interactive API documentation at ``/docs`` on a running service for the full list of curation endpoints. + Administration Endpoints -------------------------- -Administration endpoints require an **administrator token**. Use -``dump-things-upload-config`` to generate the necessary POST requests from a -configuration file (see :doc:`commands`). +A Dump Things Service instance can be reconfigured at runtime via the +administration endpoints. These endpoints allow to create, update, and delete +collections, tokens, and admin tokens. + +The service keeps an internal model of the configuration which can be modified +by adding or removing elements via the administration endpoints (see +:doc:`configuration` for a more detailed description of the configuration model). + +Note: all administration endpoints require an **administrator token**. + + +Currently the smallest units for updates are also *collections*, *tokens*, and +*admin tokens*. That means, to update, for example, the authentication sources +of an existing collection, a complete collection configuration with updated +authentication sources has to be sent. The current collection configuration +can be read via the ``GET /collections/`` endpoint. Modifying an +existing collection can therefore be implemented by reading the current +collection configuration, modifying it, and sending it to the server via the +``PUT /collections/`` endpoint. The same holds for tokens and admin tokens. + + +The commands ``dump-things-download-config`` and ``dump-things-upload-config`` +support this workflow. ``dump-things-download-config`` downloads the +current configuration from a server and stores it as configuration file. +``dump-things-upload-config`` reads a configuration file and +invokes the API requests that are necessary to create the elements from +the configuration file on a server (see :doc:`commands`). + Collections ^^^^^^^^^^^ @@ -185,7 +219,7 @@ Collections - Description * - POST - ``/collections`` - - Create a new collection. + - Create a new collection ( . * - GET - ``/collections`` - List all collections. @@ -199,6 +233,13 @@ Collections - ``/collections/`` - Remove a collection from the service state (data is not deleted). + +The payload for ``POST /collections`` and ``PUT /collections`` +is a JSON objects that corresponds to the collection configuration object +(see :ref:`config_file`), but has an additional ``name`` attribute that +specifies the name of the collection that should be created or updated. + + Tokens ^^^^^^ @@ -219,12 +260,26 @@ Tokens - ``/tokens/`` - Get information about a specific token. * - PUT - - ``/tokens/`` + - ``/tokens`` - Update an existing token. * - DELETE - ``/tokens/`` - Delete a token. +The payload for ``POST /tokens`` and ``PUT /tokens`` is a JSON objects that +corresponds to the token configuration object +(see :ref:`config_file`), but has an additional ``name`` attribute that +specifies the name of the token that should be created or updated. + +If no representation is provided in the payload, the server generates a random +token representation and returns it in the ``representation`` attribute of +the response. If a representation is +provided and ``hashed`` is ``True``, the representation must be a valid +token-hash, i.e, a sha256 hash of the plaintext-token in hexdigit format. +The command ``dump-things-hash-token`` can be used to generate a +valid token hash + + Admin Tokens ^^^^^^^^^^^^ @@ -245,8 +300,18 @@ Admin Tokens - ``/admin_tokens/`` - Get information about a specific admin token. * - PUT - - ``/admin_tokens/`` + - ``/admin_tokens`` - Update an existing admin token. * - DELETE - ``/admin_tokens/`` - Delete an admin token. + + +The payload for ``POST /admin_tokens`` and ``PUT /admin_tokens`` is a JSON +objects that corresponds to the *admin token* configuration object +(see :ref:`config_file`), but has an additional ``name`` attribute that +specifies the name of the admin token that should be created or updated. + +The ``representation`` attribute must be a valid token-hash, i.e, a sha256 hash +of the plaintext-token in hexdigit format. +The command ``dump-things-hash-token`` can be used to generate a valid token hash diff --git a/docs/introduction.rst b/docs/introduction.rst index 95b0d21..a071bd7 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -274,8 +274,4 @@ Key Features - **Flexible authentication**: config-file-based tokens and `Forgejo `_-based authentication. - **Audit logs**: track time, content, author, and curator for every change. -- **LinkML-based validation**: ensure data conforms to the defined schema. - -- **GraphQL support**: query data via a GraphQL interface. - -- **Pagination**: all list endpoints support optional pagination. +- **LinkML-based validation**: ensure data conforms to a defined schema. -- 2.52.0 From 769e322b5c32953f25147c02571d46a94c3a9577 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 16 Jul 2026 09:21:33 +0200 Subject: [PATCH 7/8] update README and readthedocs project name For consistency, the project name on readthedocs is now: `dump-things-service`. --- README.md | 944 ++------------------------------------------------- docs/conf.py | 4 +- 2 files changed, 22 insertions(+), 926 deletions(-) diff --git a/README.md b/README.md index e208c9c..eea7595 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,25 @@ [![PyPI version fury.io](https://badge.fury.io/py/dump-things-service.svg)](https://pypi.python.org/pypi/dump-things-service/) -This is an implementation of a service that allows to store and retrieve data that is structured according to given schemata. +This is an implementation of a *Dump Things Service* (DTS). The Dump Things Service +is a central component of a self-hostable research information infrastructure, which +contains tools to collect, organize, validate, visualize, and serve structured research (meta)data. + +It provides a REST API for storing and retrieving records that are structured +according to a schema (defined in LinkML). The service supports multiple storage +backends, flexible authentication sources, and a curation workflow for distributed +data acquisition. + +It is complemented by a client library and command line tool, which provide convenient +access to the service from Python code and the shell +[dump-things-pyclient](https://pypi.org/project/dump-things-pyclient/). + +Dump Things Service serves as backend for [shacl-vue](https://www.psychoinformatics.de/instruments/shacl-vue), +a generic graphical metadata acquisition tool. + +For more information on the service, see the [documentation](https://dump-things-service.readthedocs.io/). + +### Overview of the service architecture Data is stored in **collections**. Each collection has a name and an associated schema. @@ -13,932 +31,10 @@ The service supports schemas that are based on Datalad's *Thing* schema, i.e. on It assumes that the classes of stored records are subclasses of `Thing`, and inherit the properties `pid` and `schema_type` from the `Thing`-baseclass. The general workflow in the service is as follows. -We distinguish between two areas of a collection, an **incoming** area and a **curated** area. +We distinguish between two areas of a collection, an **incoming** area (aka inbox) and a **curated** area. Data written to a collection is stored in a collection-specific **incoming** area. A curation process, which is outside the scope of the service, moves data from the incoming area of a collection to the **curated** area of the collection. -To submit a record to a collection, a token is required. -The token defines read- and write- permissions for the incoming areas of collections and read-permissions for the curated area of a collection. -A token can carry permissions for multiple collections. -In addition, the token carries a submitter ID. -It also defines a token specific **zone** in the incoming area. -So any read- and write-operations on an incoming area are actually restricted to the token-specific zone in the incoming area. -Multiple tokens can share the same zone. -That allows multiple submitters to work together when storing records in the service. - -The service provides an HTTP-based API to store and retrieve data objects, and to verify token capabilities. - -### Installing the service - -The service is available via `pypi`, and can be installed by `pip`. -Execute the command `pip install dump-things-service` to install the service. - - -### Running the service - -After installation the service can be started via the command `dump-things-service`. -The basic service configuration is done via command line parameters and configuration files. - -The following command line parameters are supported: - -- ``: (mandatory) the path of a directory that serves as anchor for all relative paths given in the configuration files. Unless `-c/--config` is provided, the service will search the configuration file in `/.dumpthings.yaml`. - -- `--host `: The IP-address on which the service should accept connections (default: `0.0.0.0`). - -- `--port `: The port on which the service should accept connections (default: `8000`). - -- `-c/--config `: provide a path to a configuration file (configuration file version 1). - If no dynamically managed configuration is found in the data store, the dynamically managed configuration and the service state will be initialized with the content of the configuration file. - This allows an easy transition from dump-things-servers of version 5 and lower to version 6. - -- `--origins `: add a CORS origin hosts (repeat to add multiple CORS origin URLs).` - -- `--root-path `: Set the ASGI 'root_path' for applications submounted below a given URL path. - -- `--log-level`: set the log level for the service, allowed values are `ERROR`, `WARNING`, `INFO`, `DEBUG`. The default-level is `WARNING`. - -- `--admin-token-hash `: set an administrator token hash. - The plaintext token can be used to create and delete collections, tokens, and admin tokens. - This is useful to configure the service if no admin token was yet created. - **NOTE**: an admin token in plaintext is read from the environment variable `DTS_ADMIN_TOKEN` if it is set and this option is not provided. - - -```bash -dump-things-service /data-storage/store --host 127.0.0.1 --port 8000 -``` - -The above command runs the service on the network location `127.0.0.1:8000` and provides access to the store under `/data-storage/store`. - -### Configuration file - -The service provides the tool `dump-things-upload-config` which can load configurations from a file and manifest those configurations on a running service via the administration endpoints. - -A configuration defines collections, paths for incoming and curated data for each collection, as well as token properties. -Token properties include a submitter identification and for each collection an incoming zone specifier, permissions for reading and writing of the incoming zone and permission for reading the curated data of the collection. - -A "formal" definition of the configuration file is provided by the class `Configuration` in the file `dumpthings-server/abstract_config.py`. - -Configurations are read in YAML format. The following is an example configuration file (version 6 and higher) that illustrates all options: - -```yaml -type: collections # has to be "collections" -version: 2 # has to be 2 - -# All collections are listed in "collections" -collections: - - # The following entry defines the collection "personal_records" - personal_records: - # The token, as defined below, that is used if no token is provided by a client. - # All tokens that are provided by the client will be OR-ed with the default token. - # That means all permissions in the default token will be added to the client provided - # token. In this way the default token will always be less or equally powerful as the - # client provided token. - default_token: no_access - - # The schema that is used by the collection - schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml - - # The path to the curated data of the collection. This path should contain the - # ".dumpthings.yaml"-configuration for collections that is described - # here: . - # A relative path is interpreted relative to the storage root, which is provided on - # service start. An absolute path is interpreted as an absolute path. - curated: curated/personal_records - - # The path to the incoming data of the collection. - # Different collections should have different curated- and incoming-paths - incoming: /tmp/personal_records/incoming - - # Optionally a list of classes that should receive store- or validate-endpoints, - # if this list is present, all other classes defined in the schema will be ignored, - # i.e., they will not receive store- and validation-endpoints. The classes listed - # here must be in the schema. - use_classes: - - Organization - - Person - - Project - - Agent - - # Optionally a list of classes that will be ignored when store- or validate-endpoints - # are created. If `use_classes` is present, the entries of this list will further reduce - # the classes that will receive endpoints. If `use_classes` is not present, the entries - # of this list will reduce the classes from the schema that will receive endpoints. - # The classes listed here must be listed in `use_classes` if that is defined. If - # `use_classes` is not defined, they must be listed in the schema. - ignore_classes: - - Person - - Project - - # The following entry defines the collection "rooms_and_buildings" - rooms_and_buildings: - default_token: basic_access - curated: curated/rooms_and_buildings - incoming: incoming/rooms_and_buildings - - # The following entry defines the collection "fixed_data", which does not - # support data uploading, because there is no token that allows uploads to - # "fixed_data". - fixed_data: - default_token: basic_access - # If not upload is supported, the "incoming"-entry is not necessary. - curated: curated/fixed_data_curated - -# All tokens are listed in "tokens" -tokens: - - # The following entry defines the token "basic_access". - basic_access: - - # The representation of the token, this is the value that the user has to - # provide in the `x-dumpthings-token`-header to authenticate with this token. - representation: anonymous - - # If hashed is `True`, the representation must be a 40-hexdigit number, - # representing the hash of the plain token. Setting `hashed` to `True` ensures - # that the plain-text token is not stored in the configuration store of the - # running server. - # - # The tool `dump-things-hash-token` can be used to calculate the correct hash. - hashed: False - - # The value of "user_id" will be added as an annotation to each record that is - # uploaded with this token. - user_id: anonymous_user - - # The collections for which the token holds rights are defined in "collections" - collections: - - # The rights that "basic_access" carries for the collection "rooms_and_buildings" - # are defined here. - rooms_and_buildings: - # Access modes are defined here: - # - mode: READ_CURATED - - # A token and collection-specific label, that defines "zones" in which incoming - # records are stored. Multiple tokens can share the same zone, for example if - # many clients with individual tokens work together to build a collection. - # (Since this token does not allow write access, "incoming_label" is ignored. It - # is set to an empty string here in order to document it, but it could as well - # be omitted) - incoming_label: '' - - # The rights that "basic_access" carries for the collection "fixed_data" - # are defined here. - fixed_data: - mode: READ_CURATED - incoming_label: '' - - # The following entry defines the token "no_access". This token does not allow - # any access and is used as a default token for the collection "personal_records". - no_access: - - representation: no_access - - user_id: nobody - - collections: - personal_records: - mode: NOTHING - incoming_label: '' - - # The following entry defines a token with the name "admin_token" and the plain - # representation: "admin". It gives full access rights to the collection "personal_records". - admin_token: - representation: admin - user_id: Admin - collections: - personal_records: - mode: WRITE_COLLECTION - incoming_label: 'admin_posted_records' - - # The following entry defines the token "contributor_bob". It gives full access - # to "rooms_and_buildings" for a user with the id "Bob". - contributor_bob: - representation: bob - user_id: Bob - collections: - rooms_and_buildings: - mode: WRITE_COLLECTION - incoming_label: new_rooms_and_buildings - - # The following entry defines the token "contributor_alice". It gives full access - # to "rooms_and_buildings" for a user with the id "Alice". Bob and Alice share the - # same incoming-zone, i.e. "new_rooms_and_buildings". That means they can read - # incoming records that the other one posted. - contributor_alice: - representation: alice - user_id: Alice - collections: - rooms_and_buildings: - mode: WRITE_COLLECTION - incoming_label: new_rooms_and_buildings - - # The following entry defines a hashed token because the key `hashed` is set - # to `True`. A hashed token representation is the hex-digit representation of - # the sha-256 checksum of the plain token. - # In this example, if the client presents the token `hello`, he will be - # granted access because `dump-things-hash-token 'hello'` yields - # `90b1b286043f1b7612e423c74608f5ea2f676340507f0b67219b20d09fc4777b`, i.e. - # sha256('hello') == 90b1b286043f1b7612e423c74608f5ea2f676340507f0b67219b20d09fc4777b - # is true. - hashed_token_1: - representation: 90b1b286043f1b7612e423c74608f5ea2f676340507f0b67219b20d09fc4777b - hashed: True - user_id: Walter - collections: - rooms_and_buildings: - mode: WRITE_COLLECTION - incoming_label: bob - - # -``` - -#### Backends - -The service currently supports the following backends for storing records: -- `record_dir`: this backend stores records as YAML-files in a directory structure that is defined [here](https://concepts.datalad.org/dump-things-storage-v0/). It reads the backend configuration from a "record collection configuration file" as described [here](https://concepts.datalad.org/dump-things-storage-v0/). - -- `sqlite`: this backend stores records in a SQLite database. There is an individual database file, named `__sqlite-records.db`, for each curated area and incoming area. - -- `record_dir+stl`: here `stl` stands for "schema-type-layer". - This backend stores records in the same format as `record_dir`, but adds special treatment for the `schema_type` attribute in records. - It removes `schema_type`-attributes from the top-level mapping of a record before storing it as YAML-file. When records are read from this backend, a `schema_type` attribute is added back into the record, using a schema to determine the correct class-URI. - In other words, all records stored with this backend will have no `schema_type`-attribute in the top-level, and all records read with this backend will have a `schema_type` attribute in the top-level. - -- `sqlite+stl`: This backend stores records in the same format as `sqlite`, but adds the same special treatment for the `schema_type` attribute as `record_dir+stl`. - -Backends can be defined per collection in the configuration file. -The backend will be used for the curated area and for the incoming areas of the collection. -If no backend is defined for a collection, the `record_dir+stl`-backend is used by default. -The `+stl`-backends can be useful if an endpoint returns records of multiple classes, because it allows clients to determine the class of each result record. - -The service guarantees that backends of all types can co-exist independently in the same directory, i.e., there are no name collisions in files that are used for different backends (as long as no class name starts with `.` or `_`). - -The following configuration snippet shows how to define a backend for a collection: - -```yaml -... -collections: - collection_with_default_record_dir+stl_backend: - # This is a collection with the default backend, i.e. `record_dir+stl` and - # the default authentication, i.e. config-based authentication. - schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml - default_token: anon_read - curated: collection_1/curated - - collection_with_forgejo_authentication_source: - # This is a collection with the default backend, i.e. `record_dir+stl` and - # a forgejo-based authentication source. That means it will use a forgejo - # instance to determine the permissions of a token for this collection. - # The instance is also used to determine the user-id and the incoming label. - # In the case of forgejo, the user-id and the incoming label are the - # forgejo login associated with the token. - - # We still need the name of a default token. If the token is defined in this - # config file, its properties will be determined by the - # config file. If the token is not defined in the config file, its - # properties will be determined by the authentication sources. In this - # example by the forgejo-instance at `https://forgejo.example.com`. - # If there is more than one authentication source, they will be tried - # in the order they are defined in the config file. - schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml - default_token: anon_read # We still need a default token - curated: collection_2/curated - - # Token permissions, user-ids (for record annotations), and incoming - # label can be determined by multiple authentication sources. - # If no source is defined, `config` will be used, which reads token - # information from the config file. - # This example explicitly defines `config` and a second authentication - # source, a `forgejo` authentication source. - auth_sources: - - type: forgejo # requires `user`-read and `organization`-read permissions on token - # The API-URL of the forgejo instance that should be used - url: https://forgejo.example.com/api/v1 - # An organization - organization: data_handling - # A team in the organization. The authorization of the team - # determines the permissions of the token - team: data_entry_personal - # `label_type` determines how an incoming label is created for - # a Forgejo token. If `label_type` is `team`, the incoming label - # will be `forgejo-team--`. If `label_type` - # is `user`, the incoming label will be - # `forgejo-user-` - label_type: team - # An optional instance id. This is used to disambiguate identical - # user IDs on different Forgejo instances. If not set, a hash of - # `url` will be used instead. - instance_id: forgejo-server-1 - # An optional repository. The token will only be authorized - # if the team has access to the repository. Note: if `repository` - # is set, the token must have at least repository read - # permissions. - repository: reference-repository - - # Fallback to the config file. - - type: config # check tokens from the configuration file - - # Multiple authorization sources are allowed. They will be tried in the - # order defined in the config file. If an authorization source returns - # permissions for a token, those permissions will be used and no other - # authorization sources will be queried. - # The default authorization source is `config`, which reads the token - # permissions, user-id, and incoming from the config file. - - collection_with_explicit_record_dir+stl_backend: - schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml - default_token: anon_read - curated: collection_3/curated - backend: - # The record_dir+stl backend is identified by the - # type: "record_dir+stl". No more attributes are - # defined for this backend. - type: record_dir+stl - - collection_with_sqlite_backend: - schema: https://concepts.inm7.de/s/flat-data/unreleased.yaml - default_token: anon_read - curated: collection_4/curated - backend: - # The sqlite-backend is identified by the - # type: "sqlite". It requires a schema attribute - # that holds the URL of the schema that should - # be used in this backend. - type: sqlite -``` - -#### Reserved names - -The following collection names are reserved and must not be used: - -- collections -- tokens -- admin_tokens -- __dump_things__ - - -#### Authentication and authorization - -To authenticate and authorize a user based on tokens, dumpthing-service uses -authentication sources. There are currently two authentication sources: the -configuration file and a Forgejo-based authentication source. Authentication -sources can be configured per collection. If no authentication source is -configured, the collection uses the configuration file. - -If authentication sources are configured, they will be tried in order until -a token is authenticated. If an authentication source is listed twice, the -second instance will be ignored. - -Authentication sources can be defined individually for each collection. -The collection-level key `auth_sources` should contain a list of authentication source configurations. -Authentication sources are tried in order until a token is successfully authenticated. -If no authentication source authenticates the token, the token will be rejected. - -If no authentication source is defined, the configuration file will be used to authenticate tokens. -If an identical authentication source is defined multiple times, the first instance will be queried, all other instances will be ignored. -Authentication sources are identical if the content of their keys match. -If an identical authentication source is listed multiple time in the configuration, the service will issue a warning about `Ignoring duplicate authentication provider...`. - -These authentication sources are available: - -- config: use the configuration file to authenticate tokens -- forgejo: use a Forgejo-instance to authenticate tokens - -All authentication source configurations contain the key `type`. -Additional keys are authentication source type-specific. - -The following configuration snippet contains an example for authentication -source configuration: - -```yaml -collections: - collection_with_config_and_forgejo_auth_sources: - # Token permissions, user-ids (for record annotations), and incoming - # label can be determined by multiple authentication sources. - # If no source is defined, `config` will be used, which reads token - # information from the config file. - # This example explicitly defines `config` and a second authentication - # source, a `forgejo` authentication source. - auth_sources: - - type: forgejo # requires `user`-read and `organization`-read permissions on token - # The API-URL of the forgejo instance that should be used - url: https://forgejo.example.com/api/v1 - # An organization - organization: data_handling - # A team in the organization. The authorization of the team - # determines the permissions of the token - team: data_entry_personal - # `label_type` determines how an incoming label is created for - # a Forgejo token. If `label_type` is `team`, the incoming label - # will be `forgejo-team--`. If `label_type` - # is `user`, the incoming label will be - # `forgejo-user-` - label_type: team - # An optional repository. The token will only be authorized - # if the team has access to the repository. Note: if `repository` - # is set, the token must have at least repository read - # permissions. - repository: reference-repository - - # Fallback to the config file. - - type: config # check tokens from the configuration file - - # Multiple authorization sources are allowed. They will be tried in the - # order defined in `auth_sources`. If an authorization source returns - # permissions for a token, those permissions will be used and no other - # authorization sources will be queried. - # The default authorization source is `config`, which reads the token - # permissions, user-id, and incoming from the config file. - -... - -``` - - -##### Config-based authentication - -```yaml -collections: - collection_with_config_authentication: - default_token: anon_read - curated: collection_5/curated - auth_sources: - - type: # check tokens from the configuration file - -... -``` -The configuration file will be used to authenticate tokens - - -##### Forgejo-based authentication - -```yaml -collections: - collection_with_forgejo_authentication: - default_token: anon_read - curated: collection_5/curated - auth_sources: - - type: - url: - organization: - team: - label_type: <'team' or 'user'> - repository: # Optional -... -``` - -The defined Forgejo-instance will be used to authenticate a token - -The user ID is the email of the user. - -If `label_type` is set to `team`, the incoming label is `forgejo-team--`. -If `label_type` is set to `user`, the incoming label is `forgejo-user-` - -The permissions will be fetched from the units `repo.code` and `repo.actions` of the team definition. -The following mapping is used: - -| `repo.code` | curated_read | incoming_read | incoming_write | curated_right | zones_access | -|-------------|--------------|---------------|----------------|---------------|--------------| -| `none` | `False` | `False` | `False` | `False` | `False` | -| `read` | `True` | `True` | `False` | `False` | `False` | -| `write` | `True` | `True` | `True` | `False` | `False` | - - -| `repo.actions` | curated_read | incoming_read | incoming_write | curated_right | zones_access | -|----------------|--------------|---------------|----------------|---------------|--------------| -| `none` | `False` | `False` | `False` | `False` | `False` | -| `read` | `False` | `False` | `False` | `False` | `False` | -| `write` | `True` | `True` | `True` | `True` | `True` | - -A Forgejo authentication source can authenticate Forgejo-tokens that have at least the following `Read`-permissions: - -- User: this is required to determine user-related information, i.e. user-email and user login name. -- Organization: this is required to determine the membership of a user to a team in an organization. -- (Only if `repository` is set in the configuration) Repository : required to determine a team's access to the repository. - - -#### Submission annotation tag - -The service annotates submitted records with a submitter id and a timestamp. -Annotations consist of an annotation tag, defining the class of the annotation, and an annotation value. -By default the service will use the class `http://purl.obolibrary.org/obo/NCIT_C54269` for the submitter id and the class `http://semanticscience.org/resource/SIO_001083` for submission time. -(Both tags will be converted into CURIEs if the schema of the collection defines an appropriate prefix.) - -The default annotation tag classes can be overridden in the configuration on a per collection basis. -To override the defaults tags, add a `submission_tags`-attribute to a collection definition. -The `submission_tags`-attribute should contain a mapping that maps either `submitter_id_tag`, or `submitter_time_tag` or both to an IRI or a CURIE. -If the schema defines a matching prefix, IRIs are automatically converted to CURIEs before storing the record. -If a tag is given as a CURIE, the service validates that the prefix of the CURIE is defined in the schema of the collection. - -```yaml -type: collections -version: 1 -collections: - collection_1: - default_token: basic_access - curated: curated - incoming: contributions - submission_tags: - submitter_id_tag: schema:user_id - submission_time_tag: schema:time - - ... - -``` - -#### Audit Backends - -The service supports audit-logs of changes that are made via the curation interface. -Audit logs are configured per collection via the key `audit-backends`. -The key expects a list of audit-backend configurations. -Currently the only supported audit-backend type is `gitaudit`: - -```yaml -type: collections -version: 1 -collections: - collection_1: - - ... - - audit_backends: - - type: gitaudit - path: - auto_flush_timeout: - ... -``` -Here `` must be a path to a directory. -If the directory does not exist, it will be created. -If the directory exists, it should contain a bare git repository. - -If `auto_flush_timeout` seconds have passed without adding new audit log entries, the current changeset is persisted, resulting in a commit. -The default value for `auto_flush_timeout` is 60 seconds. The value must not be lower than `1`. - -The command `dump-things-gitaudit-report ` can be used to show the audit-log for all PIDs that match the given `PID`-pattern (pattern are in python `re`-module syntax, i.e. use `'.*'` to report changes for all PIDs). -Each log entry contains the timestamp of the change, the ID of the curator that posted the change, a diff of the change, and the resulting record. - -The command `dump-things-gitaudit-rebuild-index ` can be used to rebuild an index for a git-audit backend. -Executing this command should not be necessary in normal operations because the backend will rebuild an index if it is instantiated on a directory that has no index. -The command mainly exists for maintenance purposes. - -Note: currently the user ID of the curator will be stored as author in the audit-log entries. -The "original" author of a change is usually identified in the `annotations`-field of the record. - - -### Endpoints - -Most endpoints require a *collection*. These correspond to the names of the "data record collection"-directories (for example `myschema-v3-fmta` in [Dump Things Service](https://concepts.datalad.org/dump-things-storage-v0/)) in the stores. - -The service provides the following user endpoints (In addition to user endpoints, there exist endpoints for curators. To view them, check the `/docs`-path in an installed service): - -- `POST /maintenance`: this endpoint allows to set a collection into maintenance mode. - In maintenance mode, only tokens with curator-privileges can access the collection. - The posted data is a JSON that contains the name of the collection and whether the maintenance state should be active or not, for example: -```json -{ - "collection": "collection_1", - "active": true -} -``` - - -- `POST //record/`: an object of type `` (defined by the schema associated with ``) can be posted to this endpoint. - It will be stored in the incoming area for this collection and the user defined by the provided token. - In order to `POST` an object to the service, you MUST provide a valid token in the HTTP-header `X-DumpThings-Token` with write permissions. - The endpoint supports the query parameter `format`, to select the format of the posted data. - It can be set to `json` (the default) or to `ttl` (Terse RDF Triple Language, a.k.a. Turtle). - If the `json`-format is selected, the content-type should be `application/json`. - If the `ttl`-format is selected, the content-type should be `text/turtle`. - The service supports extraction of inlined records as described in [Dump Things Service](https://concepts.datalad.org/dump-things-storage-v0/). - On success, the endpoint will return a list of all stored records. - The list may contain more than one record if the posted object contains inlined records. - -- `POST //validate/record/`: an object of type `` (defined by the schema associated with ``) can be posted to this endpoint. - It will validate the posted data. - In order to `POST` an object to the service, you MUST provide a valid token in the HTTP-header `X-DumpThings-Token` with write permissions. - The endpoint supports the query parameter `format`, to select the format of the posted data. - It can be set to `json` (the default) or to `ttl` (Terse RDF Triple Language, a.k.a. Turtle). - If the `json`-format is selected, the content-type should be `application/json`. - If the `ttl`-format is selected, the content-type should be `text/turtle`. - The service supports extraction of inlined records as described in [Dump Things Service](https://concepts.datalad.org/dump-things-storage-v0/). - On success, the endpoint will return a list of all stored records. - The list may contain more than one record if the posted object contains inlined records. - -- `GET //records/`: retrieve all readable objects from collection `` that are of type `` or any of its subclasses. - Objects are readable if the default token or the token provided has the permission to read the objects in the collection. - Objects from incoming spaces will take precedence over objects from curated spaces, i.e. if there are two objects with identical `pid` in the curated space and in the incoming space, the object from the incoming space will be returned. - The endpoint supports the query parameter `format`, which determines the format of the query result. - It can be set to `json` (the default) or to `ttl`, - The endpoint supports the query parameter `matching`, which is interpreted by `sqlite`-backends and ignored by `record_dir`-backends. - If given, the endpoint will only return records for which the JSON-string representation matches the `matching` parameter. - Matching supports the wildcard character `%` which matches any characters. - For example, to search for `Alice` anywhere in the JSON-string representation of the record the matching parameter should be set to `%Alice%` or `%alice%` (matching is not case-sentitive). - The result is a list of JSON-records or ttl-strings, depending on the selected format. - -- `GET //records/p/`: this endpoint (ending on `.../p/`) provides the same functionality as the endpoint `GET //records/` (without `.../p/...`) but supports result pagination. In addition to the query parameters `format` and `matching`, it supports the query parameters `page` and `size`. - The `page`-parameter defines the page number to retrieve, starting with 1. - The `size`-parameter defines how many records should be returned per page. - If no `size`-parameter is given, the default value of 50 is used. - Each response will also contain the total number of records and the total number of pages in the result. - The response is a JSON object with the following structure: - ```json -{ - "items": [ ], - "total": , - "page": , - "size": , - "pages": -} -``` - -- `GET //record?pid=`: retrieve an object with the pid `` from the collection `` if the provided token allows reading. If the provided token allows reading of incoming and curated spaces, objects from incoming spaces will take precedence. - The endpoint supports the query parameter `format`, which determines the format of the query result. - It can be set to `json` (the default) or to `ttl`, - - -- `GET /server`: this endpoint provides information about the server. - The response is a JSON object with the following structure: -```json -{ - "version": "", - "collections": [ - { - "name": "collection_1", - "schema": "https://example.org/schema_1.yaml", - "classes": [ - "Thing", - "Agent", - "InstantaneousEvent", - "Person" - ] - }, - { - "name": "collection_2", - "schema": "https://example.org/schema_2.yaml" - "classes": [ - "Thing", - "AnnotationTag", - "Organization", - "Person" - ] - } - ] -} -``` - - -- `GET //records/`: retrieve all readable objects from collection ``. - Objects are readable if the default token or the token provided has the permission to read the objects in the collection. - Objects from incoming spaces will take precedence over objects from curated spaces, i.e. if there are two objects with identical `pid` in the curated space and in the incoming space, the object from the incoming space will be returned. - The endpoint supports the query parameter `format`, which determines the format of the query result. - It can be set to `json` (the default) or to `ttl`, - The endpoint supports the query parameter `matching`, which is interpreted by `sqlite`-backends and ignored by `record_dir`-backends. - If given, the endpoint will only return records for which the JSON-string representation matches the `matching` parameter. - The result is a list of JSON-records or ttl-strings, depending on the selected format. - - -- `GET //records/p/`: this endpoint (ending on `.../p/`) provides the same functionality as the endpoint `GET //records/` (without `.../p/`) but supports result pagination. In addition to the query parameters `format` and `matching`, it supports the query parameters `page` and `size`. - The `page`-parameter defines the page number to retrieve, starting with 1. - The `size`-parameter defines how many records should be returned per page. - If no `size`-parameter is given, the default value of 50 is used. - Each response will also contain the total number of records and the total number of pages in the result. - The response is a JSON object with the following structure: - ```json -{ - "items": [ ], - "total": , - "page": , - "size": , - "pages": -} -``` - - -- `DELETE //record?pid=`: delete an object with the pid `` from the incoming area of the collection `` if the provided token allows writing to the incoming area. - The result is either `True` if the object was deleted or `False` if the object did not exists or was not deleted. - - -- `GET /docs`: provides information about the service's API, i.e. about all endpoints. - -#### Curation endpoints - -The service supports a set of curation endpoints that allows direct access to the curated area as well as the incoming areas. -A `CURATOR`-token required to access these endpoints. -Details about the curation endpoints can be found in [this issue](https://codeberg.org/datalink/dump-things-server/issues/118). - - -#### Administration endpoints - -Operations on the endpoints described in this section require an administrator token. -If desired, use `dump-things-upload-config` to read the configuration from a file and -generate respective POST-requests. `dump-things-upload-config` can also be used to -generate a configuration from an old, i.e. dump-things version < 6, configuration file. - -##### Collections - -- `POST /collections`: create a new collection from the posted configuration object. - For a specification of the configuration object see the object `CollectionRequest` in the file `dump_things_service/collection_endpoints.py` - (Use `dump-things-upload-config` to read the configuration from a file and generate respective POST-requests) - -- `GET /collections`: get information about the currently existing collections. - -- `GET /collections/`: get information about the collection with name ``. - -- `DELETE /collections/`: delete the collection with the given name. - Note: deleting a collection does not delete any records or any storage dir, it just removes the - collection from the internal state of the service. Recreating it (via `POST /collections`) will make - all data reachable again through the Web-API. - - -##### Tokens - -- `POST /tokens`: create a new token from the posted configuration object. - For a specification of the configuration object see the object `TokenRequest` in the file `dump_things_service/token_endpoints.py` - NOTE: Before a token for configuration can be generated, the collection must exist. - -- `GET /tokens`: get information about the currently existing tokens. - -- `GET /tokens/`: get information about the token with name ``. - -- `DELETE /tokens/`: delete the token with the given name. - -- this endpoint (ending on `.../p/`) provides the same functionality as the endpoint `GET //records/` (without `.../p/`) but supports result pagination. In addition to the query parameters `format` and `matching`, it supports the query parameters `page` and `size`. - - -##### Admin Tokens - -- `POST /admin_tokens`: create a new admin token from the posted configuration object. - For a specification of the configuration object see the object `AdminTokenRequest` in the file `dump_things_service/token_endpoints.py` - Note that admin token are always stored as hashed values. - Therefore the representation in the request should be `sha256()` - -- `GET /admin_tokens`: get information about the currently existing admin tokens. - -- `GET /admin_tokens/`: get information about the admin token with name ``. - -- `DELETE /admin_tokens/`: delete the admin token with the given name. - - - -### Tips & Tricks - - -#### Using the same backend for incoming and curated areas - -The service can be configured in such a way that incoming records are immediately available in the curated area. -To achieve this, the final path of the incoming zone must be the same as the curated area, for example: - -```yaml -type: collections -version: 1 - -collections: - datamgt: - default_token: anon_read - curated: datamgt/curated - incoming: datamgt - -tokens: - anon_read: # The name of the token (serves also as representation if no representation is defines) - user_id: anonymous - collections: - datamgt: # per collection token configuration; contains: - mode: READ_CURATED # - token mode - incoming_label: "" # - the label for the incoming area for this token and this collection, i.e., collection: "datamgt". - - trusted-submitter-token: - user_id: trusted_submitter - representation: 00112233445566778899aabbccdd # The representation that the client has to send in an `x-dumpthings-token`-header (if not given, the token name will be the representation) - collections: - datamgt: - mode: WRITE_COLLECTION - incoming_label: "curated" -``` -In this example the curated area is `datamgt/curated` and the incoming area for the token `trusted-submitter-token` is `datamgt` plus the incoming zone `curated`, i.e., `datamgt/curated` which is exactly the curated area defined for `collection_1`. - -#### Migrating from `record_dir` (or `record_dir+stl`) to `sqlite` - -The command `dump-things-copy-store` can be used to copy a collection from a `record_dir` (or `record_dir+stl`) store to a `sqlite` store. -The command expects a source and a destination store. Both are given in the format `:`, where `` is one of `record_dir`, `record_dir+stl`, `sqlite`, or `sqlite+stl`, and `` is the path to the directory of the store. - -For example, to migrate a collection from a `record_dir`-backend at the directory `/penguis/curated` to a `sqlite` backend in the same directory, the following command can be used: -```bash -> dump-things-copy-store \ - record_dir:/penguis/curated \ - sqlite:/penguis/curated -``` - -For example, to migrate from a `record_dir+stl` backend, the command is similar, but a schema has to be supplied via the `-s/--schema` command line parameter. For example: -```bash -> dump-things-copy-store \ - --schema https://concepts.inm7.de/s/flat-data/unreleased.yaml \ - record_dir+stl:/penguis/curated \ - sqlite:/penguis/curated -``` -(Note: a `record_dir:` can be used to copy without the schema type layer from a `record_dir+stl` backend. But in this case the copied records will not have a `schema_type` attribute, because the `record_dir` backend does not "put it back in", unlike a `record_dir+stl` backend.) - -If the source backend is a `record_dir` or `record_dir+stl` backend, and the store was manually modified outside the service (for example, by adding or removing files), it is recommended to run the command `dump-things-rebuild-index` on the source store before copying. This ensures that the index is up to date and all records will be copied. - -If any backend is a `record_dir+stl` backend, a schema has to be supplied via the `-s/--schema` command line parameter. The schema is used to determine the `schema_type` attribute of the records that are copied. - - -### Maintenance commands - -- `dump-things-rebuild-index`: this command rebuilds the persistent index of a `record_dir`store. This should be done after the `record_dir` store was modified outside the service, for example, by manually adding or removing files in the directory structure of the store. - -- `dump-things-copy-store`: this command copies a collection that is stored in a source store to a destination store. For example, to copy a collection from a `record_dir` store at the directory `/penguis/curated` to a `sqlite` store in the same directory, the following command can be used: - ```bash - > dump-things-copy-store \ - record_dir:/penguis/curated \ - sqlite:/penguis/curated - ``` - The copy command will add the copied records to any existing records in the destination store. - Note: when records are copied from a `record-dir` store, the index is used to locate the records in the source store. If the index is not up-to-date, some records may not be copied. To ensure all records are copied, it is recommended to run `dump-things-rebuild-index` on the source store before copying. - -- `dump-things-pid-check`: this command checks the pids in all collections of a store to verify that they can be resolved (if they are in CURIE form). - This is useful to validate the proper definition of prefixes after schema-changes. - -- `dump-things-create-merged-schema`: this command creates a new schema that statically contains all schemas that the original schema imports. - The new schema is fully self-contained and does not reference any other schemas. - -- `dump-things-hash-token`: this command will generate a hash from a plain-text token that can be used with the `--admin-token-hash` option. - (one could also use the shell command `sha256sum` to generate the hash, but using `dump-things-hash-token` will ensure that the right hash algorithm is used) - - -### Migrate to version 6 - -Migration to version 6 is simple. It involves the following steps: - -1. Stop the old server -2. Upgrade the server to version 6 -3. Set an admin token. This can be done in one of two ways: - - provide an administrator token in hashed format on the command line - with the additional option `--admin-token-hash`. `dump-things-hash-token` can - be used to generate the hash from a plain-text token. - - Set the environment variable `DTS_ADMIN_TOKEN` to a plain-text token. - **Note**: if the command line option is provided, it takes precedence over - the environment variable. -4. Start the new server with the same path to the store as before. - -At this point the service should be running and be configured exactly as -before. The configuration is persisted in the configuration database and will -be established next time the service starts. If desired, the `--config`-option -can be removed from the command line, because the configuration is now -persisted in the configuration database and will be loaded from there on startup. - -NOTE: if the server should not be initialized from an existing default -configuration file, i.e., the file `/.dumpthings.yaml`, use the -option `--ignore-default-config-file`. If this option is provided and no -`-c/--config`-option is provided the server will start with an "empty" -configuration, i.e., without any collections or non-admin tokens. - - -### If things go wrong - -#### Delete a record manually - -If a schema is changed, for example a prefix-definition changed, the service may not be able to delete a record anymore. -In this case, the record can be deleted manually if you have access to the storage root. - -To delete the record, open a shell and navigate (`cd`) to the directory where the store is located. -The location can be determined from the configuration file. -Depending on the storage backend, the subsequent steps are different. - -##### `record-dir` backend - -Delete the record from disk by removing it, e.g. `rm -f ` - -Run the command `dump-things-rebuild-index` - -##### `sqlite` backend - -Run the command: - -```bash -> sqlite3 __sqlite-records.db -``` - -If you know the pid of the record you want to delete, enter the following on the prompt to delete the record with pid `some-pid`: - -```sql -> delete from thing where json_extract(thing.object, '$.pid') = 'some-pid'; -``` - -If you know the IRI of the record you want to delete, enter the following on the prompt to delete the record with IRI `some-iri`: - -```sql -> delete from thing where iri = 'some-iri'; -``` - - - -### Requirements - -The service requires sqlite3. - ## Acknowledgements diff --git a/docs/conf.py b/docs/conf.py index 478969f..7b849c6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,9 +16,9 @@ sys.path.insert(0, os.path.abspath('..')) import dump_things_service -project = 'dump-things-server' +project = 'dump-things-service' copyright = '2025-2026, Christian Mönch' -author = 'Christian Mönch' +author = 'Christian Mönch and' release = dump_things_service.__version__ # -- General configuration --------------------------------------------------- -- 2.52.0 From fc5685392ab4d5d94ba0ec28d717ba60cde5a213 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 16 Jul 2026 09:38:41 +0200 Subject: [PATCH 8/8] fix: fix typo --- docs/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands.rst b/docs/commands.rst index fe721d2..01ca112 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -81,7 +81,7 @@ dump-things-upload-config Read a configuration from a dump-things configuration-file and instantiate its elements on a running server via the administration -enpoints. Objects that already exist on the server are left unchanged. +endpoints. Objects that already exist on the server are left unchanged. An admin token has to be provided in the environment variable ``DTS_ADMIN_TOKEN``. -- 2.52.0