From 180c606cfcf3e406a68990a0101047ac2dfe1643 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 15:14:20 +0200 Subject: [PATCH 01/11] allows multiple inboxes to share a directory This commit disables the check for distinct directories. This allows to support some older configurations, where inboxes from different collections share a common directory. --- dump_things_service/collection_endpoints.py | 22 +++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index 6ae493d..df81e66 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -98,14 +98,20 @@ async def create_collection( detail=f"Collection name '{body.name}' is reserved and cannot be created.", ) - # Check for distinct directories - for directory in (body.incoming, body.curated): - if directory: - ensure_unique_directory( - abstract_config, - instance_state, - directory, - ) + # Check for distinct directories. + # TODO: we skip this currently because a number of version 5 installations + # deliberately put inboxes into the same path. Those configuration cannot + # be established if this check is performed. Instead of the `if False:`- + # clause, we should introduce a configuration for the server to specify + # whether unique directories are required. + if False: + for directory in (body.incoming, body.curated): + if directory: + ensure_unique_directory( + abstract_config, + instance_state, + directory, + ) # Check for incoming directory if any of the tokens allows writing validate_incoming_paths(abstract_config, body) -- 2.52.0 From 770d8fdd2b10ec55ef2ce2aaf602be8fbafb3c34 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 15:16:53 +0200 Subject: [PATCH 02/11] clean up version 2 configuration code --- dump_things_service/abstract_config.py | 4 +-- dump_things_service/commands/load_config.py | 30 ++++++++++++++------- dump_things_service/main.py | 4 +-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index b85d1b7..9f79bb6 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -159,7 +159,7 @@ class AdminTokenConfig(StrictModel): class Configuration(StrictModel): type: str = Literal['collections'] - version: str = Literal['2'] + version: int = Literal[2] collections: dict[str, CollectionConfig] = {} tokens: dict[str, TokenConfig] = {} admin_tokens: dict[str, AdminTokenConfig] = {} @@ -244,7 +244,7 @@ def read_config( if record_info else Configuration( type='collections', - version = '2', + version = 2, ) ) except ValidationError as ve: diff --git a/dump_things_service/commands/load_config.py b/dump_things_service/commands/load_config.py index 64a9983..ff1b334 100644 --- a/dump_things_service/commands/load_config.py +++ b/dump_things_service/commands/load_config.py @@ -27,13 +27,15 @@ parser.add_argument( ) parser.add_argument( '--send-to', - help='The base URL of the server API', + help='The base URL of the server API. If this option is provided, the ' + 'configuration will be sent to the server API, otherwise it will just ' + 'be written to stdout.', ) parser.add_argument( '--old-format', action='store_true', - help='If provided, assume that the configuration is in the old format ' - 'and convert it to the new format internally (in old format: tokens ' + help='If provided, assume that the configuration is in version 1 format ' + 'and convert it to the new format internally (in version 1: tokens ' 'had no `hashed`-attribute and no `representation`-attribute, the token ' 'representation was the key of the token configuration, ' 'collections had no `schema`-attribute, and `sqlite`-backends had ' @@ -56,9 +58,8 @@ def main(): with open(arguments.config_file) as config_file: configuration = yaml.safe_load(config_file) - assert configuration['type'] == 'collections', '`type`-entry missing in old config-file' if arguments.old_format: - configuration = convert_to_new_format(configuration, arguments.store) + configuration = convert_config_1_to_config_2(configuration, arguments.store) else: if arguments.store: print( @@ -68,6 +69,7 @@ def main(): flush=True, ) + assert configuration['type'] == 'collections', '`type: collections` missing in config-file' assert configuration['version'] == 2, '`version: 2` missing in config-file' if arguments.send_to: @@ -104,12 +106,20 @@ def main(): return 0 -def convert_to_new_format( +def convert_config_1_to_config_2( old_configuration: dict, store_path: str | Path, ) -> dict: - assert old_configuration['version'] == 1, '`version: 1` missing in old config-file' + old_version = old_configuration.get('version') + if old_version != 1: + msg = f'`Unknown old configuration format: {old_version}' + raise ValueError(msg) + + config_type = old_configuration.get('type') + if config_type != 'collections': + msg = f'Unknown type in config-file: {config_type}' + raise ValueError(msg) counter = count(1) new_tokens_dict = { @@ -126,7 +136,7 @@ def convert_to_new_format( for token_name, token_config in new_tokens_dict.items() } - store_path = Path(store_path) + store_path = Path(store_path) if store_path else None for collection_name, collection_config in old_configuration['collections'].items(): backend = collection_config.get('backend') if backend and backend['type'].startswith('sqlite'): @@ -146,8 +156,8 @@ def convert_to_new_format( collection_config['default_token'] = old_to_new_token_mapping[collection_config['default_token']] new_configuration = { - 'type': old_configuration['type'], - 'version': '2', + 'type': 'collections', + 'version': 2, 'tokens': new_tokens_dict, 'collections': old_configuration['collections'], 'admin_tokens': {}, diff --git a/dump_things_service/main.py b/dump_things_service/main.py index 9acf23d..c865ffb 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING from dump_things_service.abstract_config import store_config -from dump_things_service.commands.load_config import convert_to_new_format +from dump_things_service.commands.load_config import convert_config_1_to_config_2 from dump_things_service.manifest import manifest_configuration # Perform the patching before importing any third-party libraries from dump_things_service.patches import enabled # noqa F401 -- used by generated code @@ -259,7 +259,7 @@ def initialize_from_config_file( 'Converting version 1 configuration at %s', arguments.config, ) - config_dict = convert_to_new_format( + config_dict = convert_config_1_to_config_2( config_dict, instance_state.store_path, ) -- 2.52.0 From eba516f46a6c73907887bf860a956745cd5b69c0 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 15:22:02 +0200 Subject: [PATCH 03/11] rename dump-things-load-config to dump-things-upload-config --- .../commands/{load_config.py => upload_config.py} | 0 dump_things_service/main.py | 2 +- pyproject.toml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename dump_things_service/commands/{load_config.py => upload_config.py} (100%) diff --git a/dump_things_service/commands/load_config.py b/dump_things_service/commands/upload_config.py similarity index 100% rename from dump_things_service/commands/load_config.py rename to dump_things_service/commands/upload_config.py diff --git a/dump_things_service/main.py b/dump_things_service/main.py index c865ffb..7803ffc 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING from dump_things_service.abstract_config import store_config -from dump_things_service.commands.load_config import convert_config_1_to_config_2 +from dump_things_service.commands.upload_config import convert_config_1_to_config_2 from dump_things_service.manifest import manifest_configuration # Perform the patching before importing any third-party libraries from dump_things_service.patches import enabled # noqa F401 -- used by generated code diff --git a/pyproject.toml b/pyproject.toml index c8454cd..09aaf05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dump-things-pid-check = "dump_things_service.commands.check_pids:main" dump-things-create-merged-schema = "dump_things_service.commands.create_merged_schema:main" dump-things-gitaudit-report = "dump_things_service.commands.gitaudit_report:main" dump-things-gitaudit-rebuild-index = "dump_things_service.commands.gitaudit_rebuild_index:main" -dump-things-load-config = "dump_things_service.commands.load_config:main" +dump-things-upload-config = "dump_things_service.commands.upload_config:main" dump-things-hash-token = "dump_things_service.commands.hash_token:main" [tool.hatch.build.targets.wheel] -- 2.52.0 From fd147d5e71517db71012a4b56f71421ff01be343 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 15:26:51 +0200 Subject: [PATCH 04/11] adapt tests to disabled distinct directory detection --- README.md | 8 ++++---- dump_things_service/tests/test_config.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f40c3d2..ec6cc93 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ The above command runs the service on the network location `127.0.0.1:8000` and ### Configuration file -The service provides the tool `dump-things-load-config` which can load configurations from a file and manifest those configurations on a running service via the administration endpoints. +The service provides the tool `dump-things-upload-config` which can load configurations from a file and manifest those configurations on a running service via the administration endpoints. A configuration defines collections, paths for incoming and curated data for each collection, as well as token properties. Token properties include a submitter identification and for each collection an incoming zone specifier, permissions for reading and writing of the incoming zone and permission for reading the curated data of the collection. @@ -733,15 +733,15 @@ Details about the curation endpoints can be found in [this issue](https://codebe #### Administration endpoints Operations on the endpoints described in this section require an administrator token. -If desired, use `dump-things-load-config` to read the configuration from a file and -generate respective POST-requests. `dump-things-load-config` can also be used to +If desired, use `dump-things-upload-config` to read the configuration from a file and +generate respective POST-requests. `dump-things-upload-config` can also be used to generate a configuration from an old, i.e. dump-things version < 6, configuration file. ##### Collections - `POST /collections`: create a new collection from the posted configuration object. For a specification of the configuration object see the object `CollectionRequest` in the file `dump_things_service/collection_endpoints.py` - (Use `dump-things-load-config` to read the configuration from a file and generate respective POST-requests) + (Use `dump-things-upload-config` to read the configuration from a file and generate respective POST-requests) - `GET /collections`: get information about the currently existing collections. diff --git a/dump_things_service/tests/test_config.py b/dump_things_service/tests/test_config.py index 5b72fad..72efe9c 100644 --- a/dump_things_service/tests/test_config.py +++ b/dump_things_service/tests/test_config.py @@ -52,6 +52,7 @@ def test_illegal_collection_name_detection(fastapi_client_simple): assert response.status_code == HTTP_409_CONFLICT +@pytest.mark.skip(reason='Reuse detection is disabled to support existing old configurations') def test_collection_dir_reuse_detection(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple -- 2.52.0 From 4713bf545b62c9b8ab079762faab1c74c61af689 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 15:32:28 +0200 Subject: [PATCH 05/11] remove obsolete test --- dump_things_service/tests/test_basic.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/dump_things_service/tests/test_basic.py b/dump_things_service/tests/test_basic.py index 140db21..34ce9f7 100644 --- a/dump_things_service/tests/test_basic.py +++ b/dump_things_service/tests/test_basic.py @@ -282,26 +282,6 @@ def test_global_store_write_fails(fastapi_client_simple): assert response.status_code == HTTP_403_FORBIDDEN -@pytest.mark.skip(reason='No runtime store adding yet') -def test_token_store_adding(fastapi_client_simple): - test_client, store_dir = fastapi_client_simple - response = test_client.post( - '/collection_1/record/Person', - headers={'x-dumpthings-token': 'david_bowie'}, - json={'pid': extra_record['pid']}, - ) - assert response.status_code == HTTP_401_UNAUTHORIZED - - # Create collection-directory and token-directory and retry - (store_dir / 'token_stores' / 'collection_1' / 'david_bowie').mkdir() - response = test_client.post( - '/collection_1/record/Person', - headers={'x-dumpthings-token': 'david_bowie'}, - json={'pid': extra_record['pid']}, - ) - assert response.status_code == HTTP_200_OK - - def test_funky_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple record_pid = 'dlflatsocial:contributors/someone' -- 2.52.0 From 4547f702681a8f46a204ba8cb7f9806e2c26b707 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 17:00:58 +0200 Subject: [PATCH 06/11] add json format support to upload-config The command `dump-things-upload-config` now supports YAML and JSON input file format. The format is determined by the suffix or declared via the -f/--format options. --- dump_things_service/commands/upload_config.py | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index ff1b334..8e5c84b 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import sys from argparse import ArgumentParser @@ -25,6 +26,14 @@ parser.add_argument( 'config_file', help='The path to the config file', ) +parser.add_argument( + '--format', '-f', + nargs='?', + choices=['json', 'yaml'], + help='Specify the format of the input file. Possible values are `json` ' + 'and `yaml`. If this option is given, the ' + 'suffix of the configuration file is ignored.' +) parser.add_argument( '--send-to', help='The base URL of the server API. If this option is provided, the ' @@ -55,8 +64,24 @@ parser.add_argument( def main(): arguments = parser.parse_args() - with open(arguments.config_file) as config_file: - configuration = yaml.safe_load(config_file) + config_file_path = Path(arguments.config_file) + with config_file_path.open('rt') as config_file: + file_type = ( + arguments.format + if arguments.format is not None + else config_file_path.suffix[1:] + ) + if file_type == 'json': + configuration = json.load(config_file) + elif file_type == 'yaml': + configuration = yaml.safe_load(config_file) + else: + print( + f'Unknown file type {config_file_path} (use `.json` or `.yaml` suffix, or specify the format with --format )', + file=sys.stderr, + flush=True, + ) + return 1 if arguments.old_format: configuration = convert_config_1_to_config_2(configuration, arguments.store) @@ -95,14 +120,17 @@ def main(): print(f'{rte.args[0]}', file=sys.stderr, flush=True) return 2 - print( - yaml.dump( - data=configuration, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, + if config_file_path.suffix == '.json': + print(json.dumps(configuration, indent=2, sort_keys=False)) + elif config_file_path.suffix == '.yaml': + print( + yaml.dump( + data=configuration, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) ) - ) return 0 -- 2.52.0 From 1b5e9cb338e2e8616e8a1911461181d50e83debd Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Tue, 23 Jun 2026 22:07:11 +0200 Subject: [PATCH 07/11] unify result structure for all admin query endpoints The JSON-objects returned by GET-ing /collections, /tokens, and /admin_tokens can now be used verbatim to generate the respective objects by POST-ing them to the endpoints. --- dump_things_service/collection_endpoints.py | 12 +- .../commands/download_config.py | 176 ++++++++++++++++++ dump_things_service/commands/upload_config.py | 6 +- dump_things_service/token_endpoints.py | 34 +++- pyproject.toml | 1 + 5 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 dump_things_service/commands/download_config.py diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index df81e66..efdceca 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -139,14 +139,22 @@ async def create_collection( ) async def get_collections( api_key: str = Depends(api_key_header_scheme), -) -> dict[str, CollectionConfig]: +) -> list[CollectionRequest]: instance_state = get_instance_state() abstract_config = get_config() # Check admin rights authenticate_admin(instance_state, abstract_config, api_key) - return abstract_config.collections + return [ + CollectionRequest( + **{ + 'name': collection_name, + **collection_info.model_dump(mode='json', by_alias=True) + } + ) + for collection_name, collection_info in abstract_config.collections.items() + ] @router.get( diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py new file mode 100644 index 0000000..4c35271 --- /dev/null +++ b/dump_things_service/commands/download_config.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import json +import os +import sys +from argparse import ArgumentParser +from itertools import count +from pathlib import Path + +import requests +import yaml + + +parser = ArgumentParser( + prog='Download a complete configuration of a running service', + description='Read a configuration from dump-things endpoints and create a ' + 'configuration-file that can be possibly modified and uploaded ' + 'to a running service by dump-things-upload-config.' + ' ' + 'An admin token has to be provided in the environment variable ' + '`DTS_ADMIN_TOKEN`.', +) +parser.add_argument( + 'server_api', + help='The base URL of the server API.', +) +parser.add_argument( + '--entities', '-e', + action='append', + choices=['admin_tokens', 'collections', 'tokens'], + help='Specify for which entities the configuration should be downloaded. ' + ' Possible values are `admin_tokens`, `collections`, or `tokens` ' + '(repeat to download configuration for more than one entity). If this ' + 'option is not provided, configurations for all entities will be ' + 'downloaded.' +) +parser.add_argument( + '--format', '-f', + nargs='?', + default='yaml', + choices=['json', 'yaml'], + help='Specify the format of the output. Possible values are `json` ' + 'and `yaml` (the default is `yaml`).' +) + + +def main(): + arguments = parser.parse_args() + + entities = ( + arguments.entities + if arguments.entities is not None + else ['admin_tokens', 'collections', 'tokens'] + ) + + admin_token = os.environ.get('DTS_ADMIN_TOKEN') + if not admin_token: + print( + 'An admin token must be provided in the environment variable `DTS_ADMIN_TOKEN`', + file=sys.stderr, + flush=True, + ) + return 1 + + configuration = get_configuration( + arguments.server_api, + admin_token, + entities, + ) + + if arguments.format == 'json': + print(json.dumps(configuration, indent=2, sort_keys=False)) + elif arguments.format == 'yaml': + print( + yaml.dump( + data=configuration, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + ) + return 0 + + +def get_configuration( + api_url: str, + admin_token: str, + entities: list[str], +) -> dict: + + result = {} + + if 'collections' in entities: + result['collections'] = get_collections(api_url, admin_token) + + if 'tokens' in entities: + result['tokens'] = get_tokens(api_url, admin_token) + + if 'admin_tokens' in entities: + result['admin_tokens'] = get_admin_tokens(api_url, admin_token) + + return { + 'type': 'collections', + 'version': 2, + **result, + } + + +def list_to_dict_on_key( + elements: list[dict], + extract_key: str, +) -> dict: + return { + element[extract_key]: { + element_key: value for element_key, value in element.items() + if element_key != extract_key + } + for element in elements + } + + +def get_tokens( + api_url: str, + admin_token: str, +) -> dict: + token_list = _get_data( + url=api_url + '/tokens', + token=admin_token, + content_class='tokens', + ) + return list_to_dict_on_key(token_list, extract_key='name') + + +def get_collections( + api_url: str, + admin_token: str, +) -> dict: + collection_list = _get_data( + url=api_url + '/collections', + token=admin_token, + content_class='collections', + ) + return list_to_dict_on_key(collection_list, extract_key='name') + + +def get_admin_tokens( + api_url: str, + admin_token: str, +) -> dict: + admin_token_list = _get_data( + url=api_url + '/admin_tokens', + token=admin_token, + content_class='admin tokens', + ) + cleaned_admin_token_list = [ + list_entry + for list_entry in admin_token_list + if list_entry['name'] != '__bootstrap__' + ] + return list_to_dict_on_key(cleaned_admin_token_list, extract_key='name') + + +def _get_data( + url: str, + token: str, + content_class: str, +) -> list: + result = requests.get(url, headers={'x-dumpthings-token': token}) + if result.status_code >= 300: + msg = f'Error downloading {content_class}: {result.text}' + raise RuntimeError(msg) + return result.json() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index 8e5c84b..a88b7dc 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -101,7 +101,7 @@ def main(): admin_token = os.environ.get('DTS_ADMIN_TOKEN') if not admin_token: print( - 'An admin token not provided in the environment variable `DTS_ADMIN_TOKEN`', + 'An admin token must be provided in the environment variable `DTS_ADMIN_TOKEN`', file=sys.stderr, flush=True, ) @@ -120,9 +120,9 @@ def main(): print(f'{rte.args[0]}', file=sys.stderr, flush=True) return 2 - if config_file_path.suffix == '.json': + if file_type == 'json': print(json.dumps(configuration, indent=2, sort_keys=False)) - elif config_file_path.suffix == '.yaml': + elif file_type == 'yaml': print( yaml.dump( data=configuration, diff --git a/dump_things_service/token_endpoints.py b/dump_things_service/token_endpoints.py index f735e12..f8bfa15 100644 --- a/dump_things_service/token_endpoints.py +++ b/dump_things_service/token_endpoints.py @@ -1,6 +1,7 @@ import logging import random import re +from os import name from urllib.parse import quote from fastapi import ( @@ -266,10 +267,12 @@ async def create_admin_token( api_key: str = Depends(api_key_header_scheme), ): - instance_state = get_instance_state() - abstract_config = read_config(store_path=instance_state.store_path) - - authenticate_admin(instance_state, abstract_config, api_key) + # Check for conflicting token-name + if body.name == '__bootstrap__': + raise HTTPException( + status_code=HTTP_409_CONFLICT, + detail=f"The admin token name '{body.name}' is reserved and cannot be used.", + ) # Check for token content if not body.representation: @@ -280,6 +283,11 @@ async def create_admin_token( detail='Hashed token is not a 64-digits hex-number' raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) + instance_state = get_instance_state() + abstract_config = read_config(store_path=instance_state.store_path) + + authenticate_admin(instance_state, abstract_config, api_key) + # Check for existing token-name if body.name in abstract_config.admin_tokens: raise HTTPException( @@ -307,12 +315,28 @@ async def create_admin_token( ) async def get_admin_token( api_key: str = Depends(api_key_header_scheme), -) -> list[str]: +) -> list[dict]: instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) authenticate_admin(instance_state, abstract_config, api_key) + return [ + { + 'name': token_name, + **(token_value.model_dump(mode='json', by_alias=True)) + } + for token_name, token_value in abstract_config.admin_tokens.items() + ] + ( + [] + if instance_state.bootstrap_token is None + else [ + { + 'name': '__bootstrap__', + 'representation': instance_state.bootstrap_token, + } + ] + ) return list(abstract_config.admin_tokens) + ( [] if instance_state.bootstrap_token is None diff --git a/pyproject.toml b/pyproject.toml index 09aaf05..060475e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dump-things-rebuild-index = "dump_things_service.commands.rebuild_index:main" dump-things-copy-store = "dump_things_service.commands.copy_store:main" dump-things-pid-check = "dump_things_service.commands.check_pids:main" dump-things-create-merged-schema = "dump_things_service.commands.create_merged_schema:main" +dump-things-download-config = "dump_things_service.commands.download_config:main" dump-things-gitaudit-report = "dump_things_service.commands.gitaudit_report:main" dump-things-gitaudit-rebuild-index = "dump_things_service.commands.gitaudit_rebuild_index:main" dump-things-upload-config = "dump_things_service.commands.upload_config:main" -- 2.52.0 From 9100e182650b31aeffa9ae447786074ef2248c9e Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 24 Jun 2026 01:19:03 +0200 Subject: [PATCH 08/11] add PUT method for administration endpoints This commit adds a PUT method for the administration endpoints `/collections`, `/tokens`, and `/admin_tokens`. It adds a test for collection-update. --- dump_things_service/collection_endpoints.py | 37 ++++++++- .../tests/test_collection_administration.py | 83 +++++++++++++++---- dump_things_service/token_endpoints.py | 64 ++++++++++++-- 3 files changed, 158 insertions(+), 26 deletions(-) diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index efdceca..34b9dfe 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -77,6 +77,30 @@ async def create_collection( body: CollectionRequest, api_key: str = Depends(api_key_header_scheme), ): + await create_or_replace_collection(body, api_key, allow_replace=False) + response.headers['Location'] = f'/collections/{quote(body.name)}' + + +@router.put( + '/collections', + tags=['Administration interface'], + name='Create a new collection', + status_code=HTTP_201_CREATED, +) +async def replace_collection( + response: Response, + body: CollectionRequest, + api_key: str = Depends(api_key_header_scheme), +): + await create_or_replace_collection(body, api_key, allow_replace=True) + response.headers['Location'] = f'/collections/{quote(body.name)}' + + +async def create_or_replace_collection( + body: CollectionRequest, + api_key: str, + allow_replace: bool, +): instance_state = get_instance_state() abstract_config = get_config() @@ -85,7 +109,7 @@ async def create_collection( authenticate_admin(instance_state, abstract_config, api_key) # Check for existing collection name - if body.name in abstract_config.collections: + if body.name in abstract_config.collections and not allow_replace: raise HTTPException( status_code=HTTP_409_CONFLICT, detail=f"Collection with name '{body.name}' already exists.", @@ -116,6 +140,15 @@ async def create_collection( # Check for incoming directory if any of the tokens allows writing validate_incoming_paths(abstract_config, body) + # If the configuration already exist, we have to delete it here and + # manifest the reduced configuration. This ensures that the new collection + # is fully manifested later + if body.name in abstract_config.collections: + del abstract_config.collections[body.name] + # Manifest the abstract configuration + with wrap_http_exception(ConfigError): + manifest_configuration(abstract_config, instance_state) + # Update the abstract configuration abstract_config.collections[body.name] = body @@ -129,8 +162,6 @@ async def create_collection( config=abstract_config, ) - response.headers['Location'] = f'/collections/{quote(body.name)}' - @router.get( '/collections', diff --git a/dump_things_service/tests/test_collection_administration.py b/dump_things_service/tests/test_collection_administration.py index 6e5ada4..576bbc8 100644 --- a/dump_things_service/tests/test_collection_administration.py +++ b/dump_things_service/tests/test_collection_administration.py @@ -14,7 +14,7 @@ from dump_things_service import ( from dump_things_service.abstract_config import ( TokenCollectionConfig, TokenModes, - hash_token_representation, + hash_token_representation, GitAuditBackendConfig, ) from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.token_endpoints import ( @@ -78,22 +78,6 @@ def test_collection_adding(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple # Check that the collection does not yet exist - response = test_client.get( - f'/collections/{new_collection_name}', - headers={'x-dumpthings-token': admin_token}, - ) - assert response.status_code == HTTP_404_NOT_FOUND - assert not _name_in_openapi_paths(test_client, new_collection_name) - - # Add a new collection - response = test_client.post( - '/collections', - headers={'x-dumpthings-token': admin_token}, - json=new_collection_request.model_dump(mode='json', by_alias=True), - ) - assert response.status_code == HTTP_201_CREATED - assert _name_in_openapi_paths(test_client, new_collection_name) - response = test_client.get( f'/collections/{new_collection_name}', headers={'x-dumpthings-token': admin_token}, @@ -178,6 +162,71 @@ def test_collection_adding(fastapi_client_simple): assert not _name_in_openapi_paths(test_client, new_collection_name) +def test_collection_putting(fastapi_client_simple, tmp_path): + test_client, _, admin_token = fastapi_client_simple + + put_collection_name = 'test_put_collection' + put_collection_request_orig = CollectionRequest( + name=put_collection_name, + default_token='test_default_token', + curated=PurePosixPath(f'{curated}/put_test_collection'), + schema=test_schema_location, + incoming=PurePosixPath(f'{incoming}/put_test_collection'), + ) + put_collection_request_updated = CollectionRequest( + name=put_collection_name, + default_token='test_default_token', + curated=PurePosixPath(f'{curated}/put_test_collection'), + schema=test_schema_location, + incoming=PurePosixPath(f'{incoming}/put_test_collection_updated'), + audit_backends=[ + GitAuditBackendConfig( + type='gitaudit', + path=Path(tmp_path), + auto_flush_timeout=2, + ) + ] + ) + + # Check that the collection does not yet exist + response = test_client.get( + f'/collections/{put_collection_name}', + headers={'x-dumpthings-token': admin_token}, + ) + assert response.status_code == HTTP_404_NOT_FOUND + assert not _name_in_openapi_paths(test_client, put_collection_name) + + # Add the first version of the collection + response = test_client.post( + '/collections', + headers={'x-dumpthings-token': admin_token}, + json=put_collection_request_orig.model_dump(mode='json', by_alias=True), + ) + assert response.status_code == HTTP_201_CREATED + assert _name_in_openapi_paths(test_client, put_collection_name) + + audit_files = tuple(tmp_path.iterdir()) + assert len(audit_files) == 0 + + response = test_client.get( + f'/collections/{put_collection_name}', + headers={'x-dumpthings-token': admin_token}, + ) + assert response.status_code == HTTP_200_OK + + # Update the collection + response = test_client.put( + '/collections', + headers={'x-dumpthings-token': admin_token}, + json=put_collection_request_updated.model_dump(mode='json', by_alias=True), + ) + assert response.status_code == HTTP_201_CREATED + + # Check that the audit backend is activated + audit_files = tuple(tmp_path.iterdir()) + assert len(audit_files) > 0 + + def test_collection_reading(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple diff --git a/dump_things_service/token_endpoints.py b/dump_things_service/token_endpoints.py index f8bfa15..59822d9 100644 --- a/dump_things_service/token_endpoints.py +++ b/dump_things_service/token_endpoints.py @@ -77,13 +77,42 @@ async def create_token( api_key: str = Depends(api_key_header_scheme), ) -> TokenRequest: + token_request = create_or_replace_token(body, api_key, allow_replace=False) + response.headers['Location'] = f'/tokens/{quote(body.name)}' + return token_request + + +@router.put( + '/tokens', + tags=['Administration interface'], + name='Create a new token or replace a token', + status_code=HTTP_201_CREATED, +) +async def replace_token( + response: Response, + body: TokenRequest, + api_key: str = Depends(api_key_header_scheme), +) -> TokenRequest: + + token_request = create_or_replace_token(body, api_key, allow_replace=True) + response.headers['Location'] = f'/tokens/{quote(body.name)}' + return token_request + + +def create_or_replace_token( + body: TokenRequest, + api_key: str, + *, + allow_replace: bool, +) -> TokenRequest: + instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) authenticate_admin(instance_state, abstract_config, api_key) # Check for existing token-name - if body.name in abstract_config.tokens: + if body.name in abstract_config.tokens and not allow_replace: raise HTTPException( status_code=HTTP_409_CONFLICT, detail=f"Token with name '{body.name}' already exists.", @@ -160,7 +189,6 @@ async def create_token( config=abstract_config, ) - response.headers['Location'] = f'/tokens/{quote(body.name)}' return TokenRequest( name=body.name, user_id=body.user_id, @@ -266,7 +294,28 @@ async def create_admin_token( body: AdminTokenRequest, api_key: str = Depends(api_key_header_scheme), ): + return create_or_replace_admin_token(body, api_key, allow_replace=False) + +@router.put( + '/admin_tokens', + tags=['Administration interface'], + name='Add a new admin token or replace an existing token', + status_code=HTTP_201_CREATED, +) +async def replace_admin_token( + body: AdminTokenRequest, + api_key: str = Depends(api_key_header_scheme), +): + return create_or_replace_admin_token(body, api_key, allow_replace=True) + + +def create_or_replace_admin_token( + body: AdminTokenRequest, + api_key: str, + *, + allow_replace: bool, +): # Check for conflicting token-name if body.name == '__bootstrap__': raise HTTPException( @@ -290,10 +339,13 @@ async def create_admin_token( # Check for existing token-name if body.name in abstract_config.admin_tokens: - raise HTTPException( - status_code=HTTP_409_CONFLICT, - detail=f"Admin token with name '{body.name}' already exists.", - ) + if allow_replace: + del abstract_config.admin_tokens[body.name] + else: + raise HTTPException( + status_code=HTTP_409_CONFLICT, + detail=f"Admin token with name '{body.name}' already exists.", + ) # It is sufficient to add the new admin token to the admin_token dictionary # in order to manifest the new configuration. -- 2.52.0 From 47c7116066664d5ec7386784a4262f832b5e6e82 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 24 Jun 2026 01:22:44 +0200 Subject: [PATCH 09/11] use PUT to upload configurations --- dump_things_service/commands/upload_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index a88b7dc..71a2395 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -264,7 +264,7 @@ def _post_data( content_class: str, content_name: str, ): - result = requests.post(url, headers={'x-dumpthings-token': token}, json=data,) + result = requests.put(url, headers={'x-dumpthings-token': token}, json=data,) if result.status_code >= 300: msg = f'Error uploading {content_class}: {content_name}: {result.text}' raise RuntimeError(msg) -- 2.52.0 From 5e7234c6e00cfd511a1590e357a2041b20313ff3 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 24 Jun 2026 08:46:03 +0200 Subject: [PATCH 10/11] fix tests --- .../tests/test_collection_administration.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/dump_things_service/tests/test_collection_administration.py b/dump_things_service/tests/test_collection_administration.py index 576bbc8..259d375 100644 --- a/dump_things_service/tests/test_collection_administration.py +++ b/dump_things_service/tests/test_collection_administration.py @@ -12,9 +12,10 @@ from dump_things_service import ( HTTP_401_UNAUTHORIZED, ) from dump_things_service.abstract_config import ( + GitAuditBackendConfig, TokenCollectionConfig, TokenModes, - hash_token_representation, GitAuditBackendConfig, + hash_token_representation, ) from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.token_endpoints import ( @@ -78,6 +79,22 @@ def test_collection_adding(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple # Check that the collection does not yet exist + response = test_client.get( + f'/collections/{new_collection_name}', + headers={'x-dumpthings-token': admin_token}, + ) + assert response.status_code == HTTP_404_NOT_FOUND + assert not _name_in_openapi_paths(test_client, new_collection_name) + + # Add a new collection + response = test_client.post( + '/collections', + headers={'x-dumpthings-token': admin_token}, + json=new_collection_request.model_dump(mode='json', by_alias=True), + ) + assert response.status_code == HTTP_201_CREATED + assert _name_in_openapi_paths(test_client, new_collection_name) + response = test_client.get( f'/collections/{new_collection_name}', headers={'x-dumpthings-token': admin_token}, @@ -226,6 +243,14 @@ def test_collection_putting(fastapi_client_simple, tmp_path): audit_files = tuple(tmp_path.iterdir()) assert len(audit_files) > 0 + # Delete the collection again because we check for a known number of + # collections in other tests. + response = test_client.delete( + f'/collections/{put_collection_name}', + headers={'x-dumpthings-token': admin_token}, + ) + assert response.status_code == HTTP_200_OK + def test_collection_reading(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple @@ -237,7 +262,7 @@ def test_collection_reading(fastapi_client_simple): ) assert response.status_code == HTTP_200_OK response_object = response.json() - assert isinstance(response_object, dict) + assert isinstance(response_object, list) assert len(response_object) == 10 @@ -272,7 +297,8 @@ def test_admin_token_management(fastapi_client_simple): headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_200_OK - assert new_admin_token_name in response.json() + names = [entry['name'] for entry in response.json()] + assert new_admin_token_name in names # Delete the new admin token response = test_client.delete( -- 2.52.0 From df6d8a9ddbf9012a1af4195ac9f149f141e2959d Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 24 Jun 2026 15:28:52 +0200 Subject: [PATCH 11/11] allow storing of `Thing`-records When inlined record extraction was introduced, the storing of records of class `Thing` was prohibited. This restriction is lifted by this commit. --- dump_things_service/collection.py | 1 - dump_things_service/store/model_store.py | 26 ++++-- .../tests/test_extract_inline.py | 85 ++++++++++++++++++- 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 864afd7..1ef648d 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -221,7 +221,6 @@ def create_collection( active_classes &= set(collection_configuration.use_classes) if collection_configuration.ignore_classes: active_classes -= set(collection_configuration.ignore_classes) - active_classes -= {'Thing'} instance_state.collections[collection_name] = InstanceStateCollectionInfo( active_classes=active_classes, tag_info=dict(), diff --git a/dump_things_service/store/model_store.py b/dump_things_service/store/model_store.py index cb72c03..1ec971e 100644 --- a/dump_things_service/store/model_store.py +++ b/dump_things_service/store/model_store.py @@ -47,9 +47,8 @@ class _ModelStore: obj: BaseModel, submitter: str, ) -> Iterable[tuple[str, dict]]: - if obj.__class__.__name__ == 'Thing': - msg = f'Cannot store `Thing` instance: {obj}.' - raise ValueError(msg) + if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in (obj.annotations or dict()): + return [] # Extract inlined records from the object, store individual records # and return the list of stored records. @@ -142,17 +141,30 @@ class _ModelStore: *[ self.extract_inlined(sub_record) for sub_record in record.relations.values() - # Do not extract 'empty'-Thing records, those are just + # Do not extract 'empty'-Thing records with an + # `dlthings:placeholder` annotation. These records are just # placeholders for already extracted records. - if sub_record != self.model.Thing(pid=sub_record.pid) + if sub_record != self.model.Thing( + pid=sub_record.pid, + annotations={ + 'dlthings:placeholder': sub_record.pid, + }, + ) ] ) ) # Simplify the relations in this record. We use "empty" Thing objects - # as placeholders for extracted records. + # with a special annotation as placeholders for extracted records. + # Thing objects with this placeholder-annotation will never be stored + # individually. new_record = record.model_copy() new_record.relations = { - sub_record_pid: self.model.Thing(pid=sub_record_pid) + sub_record_pid: self.model.Thing( + pid=sub_record_pid, + annotations={ + 'dlthings:placeholder': sub_record_pid, + } + ) for sub_record_pid in record.relations } return [new_record, *extracted_sub_records] diff --git a/dump_things_service/tests/test_extract_inline.py b/dump_things_service/tests/test_extract_inline.py index 6cb09b4..02c7440 100644 --- a/dump_things_service/tests/test_extract_inline.py +++ b/dump_things_service/tests/test_extract_inline.py @@ -29,6 +29,7 @@ schema_path = Path(__file__).parent / 'testschema.yaml' class Thing: pid: str relations: dict[str, Thing] | None = None + annotations: dict[str, str] | None = None def model_copy(self): return copy(self) @@ -87,9 +88,24 @@ empty_inlined_object = Person( pid='dlflatsocial:test_extract_a', given_name='Opa', relations={ - 'dlflatsocial:test_extract_a_a': Thing(pid='dlflatsocial:test_extract_a_a'), - 'dlflatsocial:test_extract_a_b': Thing(pid='dlflatsocial:test_extract_a_b'), - 'dlflatsocial:test_extract_a_c': Thing(pid='dlflatsocial:test_extract_a_c'), + 'dlflatsocial:test_extract_a_a': Thing( + pid='dlflatsocial:test_extract_a_a', + annotations={ + 'dlthings:placeholder': 'dlflatsocial:test_extract_a_a', + }, + ), + 'dlflatsocial:test_extract_a_b': Thing( + pid='dlflatsocial:test_extract_a_b', + annotations={ + 'dlthings:placeholder': 'dlflatsocial:test_extract_a_b', + }, + ), + 'dlflatsocial:test_extract_a_c': Thing( + pid='dlflatsocial:test_extract_a_c', + annotations={ + 'dlthings:placeholder': 'dlflatsocial:test_extract_a_c', + }, + ), }, ) @@ -333,3 +349,66 @@ def test_dont_extract_empty_things_on_service(fastapi_client_simple): json=empty_inlined_json_record, ) assert response.status_code == HTTP_200_OK + + +def test_store_things(fastapi_client_simple): + test_client, store, _ = fastapi_client_simple + + simple_thing = { + 'pid': 'http://test.simple.thing/1', + } + # Deposit JSON record + response = test_client.post( + '/collection_1/record/Thing', + headers={'x-dumpthings-token': 'token-1'}, + json=simple_thing, + ) + assert response.status_code == HTTP_200_OK + + # Try to read it back + response = test_client.get( + f'/collection_1/record?pid={simple_thing["pid"]}', + headers={'x-dumpthings-token': 'token-1'}, + ) + assert response.status_code == HTTP_200_OK + assert response.json()['pid'] == 'http://test.simple.thing/1' + + +def test_store_complex_things(fastapi_client_simple): + test_client, store, _ = fastapi_client_simple + + complex_thing = { + 'pid': 'http://test.complex.thing/1', + 'relations': { + 'http://test.complex.thing/1.1': { + 'pid': 'http://test.complex.thing/1.1', + 'relations': { + 'http://test.complex.thing/1.1.1': { + 'pid': 'http://test.complex.thing/1.1.1', + } + } + } + } + } + + # Deposit JSON record + response = test_client.post( + '/collection_1/record/Thing', + headers={'x-dumpthings-token': 'token-1'}, + json=complex_thing, + ) + assert response.status_code == HTTP_200_OK + assert len(response.json()) == 3 + + # Try to read individual extracted elements + for pid in ( + 'http://test.complex.thing/1', + 'http://test.complex.thing/1.1', + 'http://test.complex.thing/1.1.1', + ): + response = test_client.get( + f'/collection_1/record?pid={pid}', + headers={'x-dumpthings-token': 'token-1'}, + ) + assert response.status_code == HTTP_200_OK + assert response.json()['pid'] == pid -- 2.52.0