From 719e78a80725a776316e715206c65a35aaa06506 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 1 Jul 2026 23:51:53 +0200 Subject: [PATCH 1/9] fix: ensure sorting of yaml-keys --- dump_things_service/abstract_config.py | 2 +- dump_things_service/audit/gitaudit.py | 11 ++++------- dump_things_service/backends/record_dir.py | 9 +++------ dump_things_service/commands/download_config.py | 11 +++-------- dump_things_service/commands/upload_config.py | 10 ++-------- dump_things_service/utils.py | 12 ++++++++++++ 6 files changed, 25 insertions(+), 30 deletions(-) diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 6ec9f16..21c8dad 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -270,7 +270,7 @@ def store_config( global g_abstract_configuration # noqa PLW0603 -- this function updates a globally referenced instance config_backend, audit_backend = get_config_backends(store_path) - json_object = config.model_dump(mode='json', exclude_none=True, by_alias=True) + json_object = config.model_dump(mode='json', exclude_unset=True, by_alias=True) json_object['pid'] = dump_things_config_iri config_backend.add_record( iri=dump_things_config_iri, diff --git a/dump_things_service/audit/gitaudit.py b/dump_things_service/audit/gitaudit.py index 926babc..382b05d 100644 --- a/dump_things_service/audit/gitaudit.py +++ b/dump_things_service/audit/gitaudit.py @@ -223,14 +223,11 @@ class GitAuditBackend(AuditBackend): author_id: str, record: dict, ) -> bool: + from dump_things_service.utils import json2yaml # noqa PLC0415 -- global import leads to circular imports + existing_record = self._read_record_from_repo_path(location[1]) if existing_record != record: - self.current_change_set[location[1]] = yaml.dump( - data=record, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) + self.current_change_set[location[1]] = json2yaml(record) self._add_log_entry(location[2], committer_id, author_id) self._add_index_entry(record_id) return True @@ -249,7 +246,7 @@ class GitAuditBackend(AuditBackend): 'author_id': author_id, } log_content = self._read_from_repo_path(log_location).decode() - log_content += json.dumps(entry, ensure_ascii=False) + '\n' + log_content += json.dumps(entry, ensure_ascii=False, sort_keys=True) + '\n' self.current_change_set[log_location] = log_content def _add_index_entry( diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 1af59db..802f8f5 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -111,6 +111,8 @@ class _RecordDirStore(StorageBackend): class_name: str, json_object: dict, ): + from dump_things_service.utils import json2yaml # noqa PLC0415 -- global import leads to circular imports + pid = json_object['pid'] # Generate the class directory, apply the mapping function to the record @@ -132,12 +134,7 @@ class _RecordDirStore(StorageBackend): storage_path.parent.mkdir(parents=True, exist_ok=True) # Convert the record object into a YAML object - data = yaml.dump( - data=json_object, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) + data = json2yaml(json_object) storage_path.write_text(data, encoding='utf-8') # Add the IRI to the index. diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py index 3517535..2f36124 100644 --- a/dump_things_service/commands/download_config.py +++ b/dump_things_service/commands/download_config.py @@ -9,6 +9,8 @@ import requests import yaml from starlette.status import HTTP_300_MULTIPLE_CHOICES +from dump_things_service.utils import json2yaml + parser = ArgumentParser( prog='Download a complete configuration of a running service', description='Read a configuration from dump-things endpoints and create a ' @@ -76,14 +78,7 @@ def main(): if arguments.format == 'json': print(json.dumps(configuration, indent=2, sort_keys=False)) # noqa T201 -- cli result output elif arguments.format == 'yaml': - print( # noqa T201 -- cli result output - yaml.dump( - data=configuration, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) - ) + print(json2yaml(configuration)) # noqa T201 -- cli result output return 0 diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index f776e3f..cdd4abb 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -12,6 +12,7 @@ import yaml from starlette.status import HTTP_300_MULTIPLE_CHOICES from dump_things_service.instance_state import get_record_dir_config +from dump_things_service.utils import json2yaml CONFIG_VERSION = 2 @@ -129,14 +130,7 @@ def main(): if file_type == 'json': print(json.dumps(configuration, indent=2, sort_keys=False)) # noqa T201 -- cli result output elif file_type == 'yaml': - print( # noqa T201 -- cli result output - yaml.dump( - data=configuration, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) - ) + print(json2yaml(configuration)) # noqa T201 -- cli result output return 0 diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index 2dcb210..be34076 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -18,6 +18,7 @@ from typing import ( ) import fsspec +import yaml from fastapi import HTTPException from rdflib import Graph @@ -527,3 +528,14 @@ def var_escape( name: str, ) -> str: return name.replace('_', '___').replace('-', '_0_') + + +def json2yaml( + json: dict, +) -> str: + return yaml.dump( + data=json, + sort_keys=True, + allow_unicode=True, + default_flow_style=False, + ) -- 2.52.0 From 80ce5019bae88a8f9d4ba99f3fe695ca56cca18e Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 2 Jul 2026 00:07:52 +0200 Subject: [PATCH 2/9] fix: ensure sorting of JSON-objects in sqlite-backend --- dump_things_service/backends/sqlite.py | 4 +++- dump_things_service/utils.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index 31049c2..5fee9f1 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -55,6 +55,7 @@ from dump_things_service.backends import ( StorageBackend, create_sort_key, ) +from dump_things_service.utils import order_dict if TYPE_CHECKING: from collections.abc import Iterable @@ -170,12 +171,13 @@ class _SQLiteBackend(StorageBackend): class_name: str, json_object: dict, ): + ordered_json_object = order_dict(json_object) with Session(self.engine) as session, session.begin(): self._add_record_with_session( session=session, iri=iri, class_name=class_name, - json_object=json_object, + json_object=ordered_json_object, ) def add_records_bulk( diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index be34076..3612d5b 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -539,3 +539,16 @@ def json2yaml( allow_unicode=True, default_flow_style=False, ) + + +def order_dict( + d: dict | list | str | int | None, +) -> dict | list | str | int | None: + + if isinstance(d, dict): + return { + k: order_dict(d[k]) for k in sorted(d) + } + elif isinstance(d, list): + return [order_dict(e) for e in d] + return d -- 2.52.0 From c73d24cafb2e5808be4f148bff5feefaca718625 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Thu, 2 Jul 2026 13:37:39 +0200 Subject: [PATCH 3/9] feat: sort JSON records before saving them This PR moves JSON-record sorting to the layer above the model-store, i.e., to the API-layer. This ensures that model-store entities and audit-backends receive identical JSON records with identical key-order. --- dump_things_service/abstract_config.py | 8 ++- dump_things_service/backends/record_dir.py | 4 +- .../backends/record_dir_index.py | 1 + dump_things_service/backends/sqlite.py | 5 +- dump_things_service/collection.py | 6 +- .../commands/download_config.py | 1 - dump_things_service/curated.py | 27 +++++--- dump_things_service/incoming.py | 61 ++++++++++++------- dump_things_service/instance_state.py | 13 ++++ dump_things_service/tests/fixtures.py | 7 ++- dump_things_service/tests/test_canonical.py | 23 +++++++ dump_things_service/utils.py | 2 +- 12 files changed, 116 insertions(+), 42 deletions(-) create mode 100644 dump_things_service/tests/test_canonical.py diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 21c8dad..7d897d5 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -267,18 +267,20 @@ def store_config( store_path, config: Configuration, ): + from dump_things_service.utils import order_dict # noqa PLC0415 -- global import leads to circular imports global g_abstract_configuration # noqa PLW0603 -- this function updates a globally referenced instance config_backend, audit_backend = get_config_backends(store_path) - json_object = config.model_dump(mode='json', exclude_unset=True, by_alias=True) + json_object = config.model_dump(mode='json', exclude_none=True, by_alias=True) json_object['pid'] = dump_things_config_iri + sorted_json_object = order_dict(json_object) config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object=json_object, + json_object=sorted_json_object, ) audit_backend.add_record( - record=json_object, + record=sorted_json_object, committer_id='__dump_things_server__', ) g_abstract_configuration = config diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 802f8f5..3fe67d5 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -61,12 +61,14 @@ class RecordDirResultList(BackendResultList): :param path: The path where the record is stored :return: A RecordInfo object. """ + from dump_things_service.utils import order_dict # noqa PLC0415 -- global import leads to circular imports + with path.open('r') as f: json_object = yaml.load(f, Loader=yaml.SafeLoader) return RecordInfo( iri=iri, class_name=class_name, - json_object=json_object, + json_object=order_dict(json_object), sort_key=sort_key, ) diff --git a/dump_things_service/backends/record_dir_index.py b/dump_things_service/backends/record_dir_index.py index 510d49d..723ee72 100644 --- a/dump_things_service/backends/record_dir_index.py +++ b/dump_things_service/backends/record_dir_index.py @@ -41,6 +41,7 @@ if TYPE_CHECKING: __all__ = [ 'IndexEntry', 'RecordDirIndex', + 'index_file_name', ] index_file_name = '.directory_dir_index.db' diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index 5fee9f1..e0bc3df 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -55,7 +55,6 @@ from dump_things_service.backends import ( StorageBackend, create_sort_key, ) -from dump_things_service.utils import order_dict if TYPE_CHECKING: from collections.abc import Iterable @@ -65,6 +64,7 @@ if TYPE_CHECKING: __all__ = [ 'SQLiteBackend', '_SQLiteBackend', + 'record_file_name', ] logger = logging.getLogger('dump_things_service') @@ -171,13 +171,12 @@ class _SQLiteBackend(StorageBackend): class_name: str, json_object: dict, ): - ordered_json_object = order_dict(json_object) with Session(self.engine) as session, session.begin(): self._add_record_with_session( session=session, iri=iri, class_name=class_name, - json_object=ordered_json_object, + json_object=json_object, ) def add_records_bulk( diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 6be11ab..6ecb02e 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -80,6 +80,7 @@ from dump_things_service.utils import ( create_store, get_token_store, join_default_token_permissions, + order_dict, var_escape, wrap_http_exception, ) @@ -665,9 +666,10 @@ def store_record( ): instance_state.validators[collection].validate(record) + sorted_record = order_dict(record) with wrap_http_exception(CurieResolutionError): stored_records = store.store_object( - obj=record, + obj=sorted_record, submitter=user_id if add_submission_tag else None, ) @@ -690,4 +692,4 @@ def store_record( ), media_type='text/turtle', ) - return JSONResponse([record for _, record in stored_records]) + return JSONResponse([order_dict(record) for _, record in stored_records]) diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py index 2f36124..65226c1 100644 --- a/dump_things_service/commands/download_config.py +++ b/dump_things_service/commands/download_config.py @@ -6,7 +6,6 @@ import sys from argparse import ArgumentParser import requests -import yaml from starlette.status import HTTP_300_MULTIPLE_CHOICES from dump_things_service.utils import json2yaml diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index bfb6976..4261b1c 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -13,6 +13,7 @@ from fastapi_pagination import ( add_pagination, paginate, ) +from starlette.status import HTTP_403_FORBIDDEN from dump_things_service import ( HTTP_401_UNAUTHORIZED, @@ -26,12 +27,14 @@ from dump_things_service.abstract_config import ( from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError -from dump_things_service.instance_state import get_instance_state +from dump_things_service.instance_state import get_instance_state, \ + check_instance_state_collection from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, check_bounds, cleaned_json, + order_dict, wrap_http_exception, ) @@ -56,7 +59,7 @@ async def {name}( repr(author_id), repr({model_var_name}), ) - return await store_curated_record( + return store_curated_record( '{collection}', data, '{class_name}', @@ -83,6 +86,7 @@ async def read_curated_records_of_type( matching: str | None = None, ): instance_state = get_instance_state() + check_instance_state_collection(instance_state, collection) if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( status_code=HTTP_404_NOT_FOUND, @@ -111,6 +115,7 @@ async def read_curated_records_of_type_paginated( matching: str | None = None, ) -> Page[dict]: instance_state = get_instance_state() + check_instance_state_collection(instance_state, collection) if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( status_code=HTTP_404_NOT_FOUND, @@ -218,6 +223,13 @@ async def _read_curated_records( if record_info: return record_info.json_object return None + # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of + # returning an empty HTTP_200_OK result. This will influence on how the + # endpoint is used by the clients. + #raise HTTPException( + # status_code=HTTP_404_NOT_FOUND, + # detail=f"no record with pid '{pid}' in curated area of collection '{collection}'", + #) if class_name: result_list = backend.get_records_of_classes([class_name], matching) else: @@ -276,7 +288,7 @@ def _get_store_and_backend( permissions = auth_info.token_permission if permissions.curated_write is False: raise HTTPException( - status_code=HTTP_401_UNAUTHORIZED, + status_code=HTTP_403_FORBIDDEN, detail=f'no write access to curated area of collection `{collection}`', ) @@ -292,8 +304,8 @@ def store_curated_record( collection: str, data: BaseModel, class_name: str, - author_id: str | None = None, - api_key: str | None = Depends(api_key_header_scheme), + author_id: str | None, + api_key: str | None, ): instance_state = get_instance_state() with wrap_http_exception( @@ -311,16 +323,17 @@ def store_curated_record( remove_keys=('@type',), ) + sorted_json_object = order_dict(json_object) with wrap_http_exception(CurieResolutionError): backend.add_record( model_store.pid_to_iri(pid), class_name, - json_object, + sorted_json_object, ) for audit_backend in instance_state.audit_backends[collection]: audit_backend.add_record( - record=json_object, + record=sorted_json_object, committer_id=auth_info.user_id, author_id=author_id, ) diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index 0de564e..c3d53c5 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging from typing import TYPE_CHECKING, Annotated from fastapi import ( @@ -13,6 +12,7 @@ from fastapi_pagination import ( add_pagination, paginate, ) +from starlette.status import HTTP_403_FORBIDDEN from dump_things_service import ( HTTP_401_UNAUTHORIZED, @@ -28,7 +28,8 @@ from dump_things_service.abstract_config import ( from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError -from dump_things_service.instance_state import get_instance_state +from dump_things_service.instance_state import get_instance_state, \ + check_instance_state_collection, InstanceState from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, @@ -36,6 +37,7 @@ from dump_things_service.utils import ( cleaned_json, create_token_store, get_on_disk_labels, + order_dict, wrap_http_exception, ) @@ -47,7 +49,6 @@ if TYPE_CHECKING: from dump_things_service.store.model_store import _ModelStore -logger = logging.getLogger('dump_things_service') router = APIRouter() add_pagination(router) @@ -69,7 +70,7 @@ async def incoming_read_labels( on_disk_labels = get_on_disk_labels( instance_state.store_path, get_config(), collection ) - return list(configured_labels.union(on_disk_labels)) + return sorted(list(configured_labels.union(on_disk_labels))) @router.get( @@ -85,11 +86,7 @@ async def incoming_read_records_of_type( matching: str | None = None, ): instance_state = get_instance_state() - if class_name not in instance_state.collections[collection].active_classes: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No '{class_name}'-class in collection '{collection}'.", - ) + _check_collection_and_class(instance_state, collection, class_name) return await _incoming_read_records( collection=collection, @@ -115,11 +112,7 @@ async def incoming_read_records_of_type_paginated( matching: str | None = None, ) -> Page[dict]: instance_state = get_instance_state() - if class_name not in instance_state.collections[collection].active_classes: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No '{class_name}'-class in collection '{collection}'.", - ) + _check_collection_and_class(instance_state, collection, class_name) record_list = await _incoming_read_records( collection=collection, @@ -128,10 +121,24 @@ async def incoming_read_records_of_type_paginated( pid=None, matching=matching, api_key=api_key, + upper_bound=None, ) return paginate(record_list) +def _check_collection_and_class( + instance_state: InstanceState, + collection: str, + class_name: str, +): + check_instance_state_collection(instance_state, collection) + if class_name not in instance_state.collections[collection].active_classes: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No '{class_name}'-class in collection '{collection}'.", + ) + + @router.get( '/{collection}/incoming/{label}/records/', tags=['Incoming area: read records'], @@ -223,7 +230,7 @@ async def _incoming_read_records( pid: str | None, matching: str | None = None, api_key: str | None = None, - upper_bound: int = 1000, + upper_bound: int | None = 1000, ) -> LazyList | dict | None: model_store, backend = await _get_store_and_backend(collection, label, api_key) @@ -232,6 +239,13 @@ async def _incoming_read_records( if record_info: return record_info.json_object return None + # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of + # returning an empty HTTP_200_OK result. This will influence on how the + # endpoint is used by the clients. + #raise HTTPException( + # status_code=HTTP_404_NOT_FOUND, + # detail=f'no record with pid: {pid}', + #) if class_name: result_list = backend.get_records_of_classes([class_name], matching) else: @@ -242,9 +256,9 @@ async def _incoming_read_records( len(result_list), upper_bound, collection, - f'/incoming/{label}/records/p/{class_name}' + f'/incoming/{label}/records/{class_name}' if class_name - else f'/incoming/{label}/records/p/', + else f'/incoming/{label}/records/', ) return ModifierList( @@ -256,7 +270,7 @@ async def _incoming_read_records( async def _incoming_delete_record( collection: str, label: str, - pid: str | None, + pid: str, api_key: str | None = None, ) -> bool: model_store, backend = await _get_store_and_backend(collection, label, api_key) @@ -274,10 +288,10 @@ async def _incoming_delete_record( async def _get_store_and_backend( collection: str, label: str, - plain_token: str | None, + api_key: str | None, ) -> tuple[_ModelStore, StorageBackend]: # Authorize api_key - await authorize_zones(collection, plain_token) + await authorize_zones(collection, api_key) # Check that the incoming zone exists instance_state = get_instance_state() @@ -327,7 +341,7 @@ async def authorize_zones( permissions = auth_info.token_permission if permissions.zones_access is False: raise HTTPException( - status_code=HTTP_401_UNAUTHORIZED, + status_code=HTTP_403_FORBIDDEN, detail=f'no access to incoming zones of collection `{collection}`', ) @@ -337,7 +351,7 @@ async def store_incoming_record( label: str, data: BaseModel, class_name: str, - api_key: str | None = Depends(api_key_header_scheme), + api_key: str | None, ): instance_state = get_instance_state() with wrap_http_exception( @@ -359,9 +373,10 @@ async def store_incoming_record( remove_keys=('@type',), ) + sorted_json_object = order_dict(json_object) with wrap_http_exception(CurieResolutionError): backend.add_record( model_store.pid_to_iri(pid), class_name, - json_object, + sorted_json_object, ) diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index f634ace..f2d3c89 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -11,7 +11,9 @@ from typing import ( import yaml from pydantic import ValidationError from yaml.scanner import ScannerError +from fastapi import HTTPException +from dump_things_service import HTTP_404_NOT_FOUND from dump_things_service.abstract_config import ( MappingMethod, RecordDirConfigFileContent, @@ -155,3 +157,14 @@ def get_mapping_function_by_name(mapping_function_name: str) -> Callable: def get_mapping_function(collection_config: RecordDirConfigFileContent): return mapping_functions[collection_config.idfx] + + +def check_instance_state_collection( + instance_state: InstanceState, + collection: str, +): + if collection not in instance_state.collections: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No such collection: '{collection}'.", + ) diff --git a/dump_things_service/tests/fixtures.py b/dump_things_service/tests/fixtures.py index 39bd595..a9f02dd 100644 --- a/dump_things_service/tests/fixtures.py +++ b/dump_things_service/tests/fixtures.py @@ -395,6 +395,7 @@ def fastapi_client_simple(fastapi_app_simple): response = test_client.post( '/collections', json=collection_config.model_dump( + exclude_none=True, exclude_unset=True, mode='json', by_alias=True, @@ -407,7 +408,11 @@ def fastapi_client_simple(fastapi_app_simple): for token_config in g_default_tokens: response = test_client.post( '/tokens', - json=token_config.model_dump(exclude_unset=True, mode='json'), + json=token_config.model_dump( + exclude_none=True, + exclude_unset=True, + mode='json', + ), headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == 201 diff --git a/dump_things_service/tests/test_canonical.py b/dump_things_service/tests/test_canonical.py new file mode 100644 index 0000000..867a480 --- /dev/null +++ b/dump_things_service/tests/test_canonical.py @@ -0,0 +1,23 @@ +from dump_things_service import HTTP_200_OK + + +def test_canonicalization(fastapi_client_simple): + test_client, _, _ = fastapi_client_simple + + record_a = { + 'pid': 'http://example.com/test_canonicalization/1', + 'given_name': 'Alice', + } + + record_b = { + 'given_name': 'Alice', + 'pid': 'http://example.com/test_canonicalization/1', + } + + for record in (record_a, record_b): + response = test_client.post( + '/collection_1/record/Person', + headers={'x-dumpthings-token': 'token-1'}, + json=record, + ) + assert response.status_code == HTTP_200_OK diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index 3612d5b..e497851 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -549,6 +549,6 @@ def order_dict( return { k: order_dict(d[k]) for k in sorted(d) } - elif isinstance(d, list): + if isinstance(d, list): return [order_dict(e) for e in d] return d -- 2.52.0 From a6ecfdcc5f15b9b274a9ddcccfa71155c6129d95 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Fri, 3 Jul 2026 11:54:25 +0200 Subject: [PATCH 4/9] feat: add canonicalisation tests --- dump_things_service/tests/test_canonical.py | 41 +++++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/dump_things_service/tests/test_canonical.py b/dump_things_service/tests/test_canonical.py index 867a480..bf8d717 100644 --- a/dump_things_service/tests/test_canonical.py +++ b/dump_things_service/tests/test_canonical.py @@ -1,19 +1,32 @@ +from pathlib import Path + from dump_things_service import HTTP_200_OK +from dump_things_service.abstract_config import ( + MappingMethod, + mapping_functions, +) def test_canonicalization(fastapi_client_simple): - test_client, _, _ = fastapi_client_simple + test_client, store_path, _admin_token = fastapi_client_simple + pid = 'http://example.com/test_canonicalization/1' record_a = { - 'pid': 'http://example.com/test_canonicalization/1', + 'pid': pid, 'given_name': 'Alice', } record_b = { - 'given_name': 'Alice', - 'pid': 'http://example.com/test_canonicalization/1', + 'given_name': 'Bob', + 'pid': pid, } + # Ensure that the two records are represented in the same order on disk. + # (The current implementation should guarantee that because the pydantic + # objects that are instantiated from the JSON input have a fixed order + # of attributes. Once the implementation is changed, this test here might + # catch regressions). + file_content = [] for record in (record_a, record_b): response = test_client.post( '/collection_1/record/Person', @@ -21,3 +34,23 @@ def test_canonicalization(fastapi_client_simple): json=record, ) assert response.status_code == HTTP_200_OK + file_content.append( + _get_record_dir_record( + store_path, + 'incoming/collection_1/in_token_1/Person', + pid, + MappingMethod.digest_md5, + ), + ) + assert file_content[0] == file_content[1].replace('Bob', 'Alice') + + +def _get_record_dir_record( + store_path: Path, + path_part: str | Path, + pid: str, + mapping_method: MappingMethod, +) -> str: + file_name = mapping_functions[mapping_method](pid, 'yaml') + with (store_path / path_part / file_name).open('rt') as f: + return f.read() -- 2.52.0 From a79936ea0da00a8e0d001e960e98ada165ebd4b1 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Fri, 3 Jul 2026 13:09:24 +0200 Subject: [PATCH 5/9] chore: adapt linter suggestions --- dump_things_service/curated.py | 6 ++++-- dump_things_service/incoming.py | 9 ++++++--- dump_things_service/instance_state.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index 4261b1c..a0fd358 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -27,8 +27,10 @@ from dump_things_service.abstract_config import ( from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError -from dump_things_service.instance_state import get_instance_state, \ - check_instance_state_collection +from dump_things_service.instance_state import ( + check_instance_state_collection, + get_instance_state, +) from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index c3d53c5..a9018ce 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -28,8 +28,11 @@ from dump_things_service.abstract_config import ( from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError -from dump_things_service.instance_state import get_instance_state, \ - check_instance_state_collection, InstanceState +from dump_things_service.instance_state import ( + InstanceState, + check_instance_state_collection, + get_instance_state, +) from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, @@ -70,7 +73,7 @@ async def incoming_read_labels( on_disk_labels = get_on_disk_labels( instance_state.store_path, get_config(), collection ) - return sorted(list(configured_labels.union(on_disk_labels))) + return sorted(configured_labels.union(on_disk_labels)) @router.get( diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index f2d3c89..26df181 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -9,9 +9,9 @@ from typing import ( ) import yaml +from fastapi import HTTPException from pydantic import ValidationError from yaml.scanner import ScannerError -from fastapi import HTTPException from dump_things_service import HTTP_404_NOT_FOUND from dump_things_service.abstract_config import ( -- 2.52.0 From d5d62a123d44f8dffd00872b7db0f8b0a20a17ca Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Fri, 3 Jul 2026 13:10:37 +0200 Subject: [PATCH 6/9] chore: format according to `hatch fmt` --- dump_things_service/abstract_config.py | 1 + dump_things_service/curated.py | 4 ++-- dump_things_service/incoming.py | 4 ++-- dump_things_service/instance_state.py | 4 ++-- dump_things_service/utils.py | 7 ++----- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 7d897d5..1eba71b 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -268,6 +268,7 @@ def store_config( config: Configuration, ): from dump_things_service.utils import order_dict # noqa PLC0415 -- global import leads to circular imports + global g_abstract_configuration # noqa PLW0603 -- this function updates a globally referenced instance config_backend, audit_backend = get_config_backends(store_path) diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index a0fd358..d2c5b43 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -228,10 +228,10 @@ async def _read_curated_records( # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of # returning an empty HTTP_200_OK result. This will influence on how the # endpoint is used by the clients. - #raise HTTPException( + # raise HTTPException( # status_code=HTTP_404_NOT_FOUND, # detail=f"no record with pid '{pid}' in curated area of collection '{collection}'", - #) + # ) if class_name: result_list = backend.get_records_of_classes([class_name], matching) else: diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index a9018ce..ed94359 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -245,10 +245,10 @@ async def _incoming_read_records( # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of # returning an empty HTTP_200_OK result. This will influence on how the # endpoint is used by the clients. - #raise HTTPException( + # raise HTTPException( # status_code=HTTP_404_NOT_FOUND, # detail=f'no record with pid: {pid}', - #) + # ) if class_name: result_list = backend.get_records_of_classes([class_name], matching) else: diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index 26df181..a77bb64 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -160,8 +160,8 @@ def get_mapping_function(collection_config: RecordDirConfigFileContent): def check_instance_state_collection( - instance_state: InstanceState, - collection: str, + instance_state: InstanceState, + collection: str, ): if collection not in instance_state.collections: raise HTTPException( diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index e497851..c2d9a9a 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -542,13 +542,10 @@ def json2yaml( def order_dict( - d: dict | list | str | int | None, + d: dict | list | str | int | None, ) -> dict | list | str | int | None: - if isinstance(d, dict): - return { - k: order_dict(d[k]) for k in sorted(d) - } + return {k: order_dict(d[k]) for k in sorted(d)} if isinstance(d, list): return [order_dict(e) for e in d] return d -- 2.52.0 From 26aabca4b07e873d94ca9a5a2fdccca4604c8d1d Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Fri, 3 Jul 2026 15:43:41 +0200 Subject: [PATCH 7/9] fix: fix type errors --- dump_things_service/abstract_config.py | 8 +-- dump_things_service/auth/__init__.py | 2 +- dump_things_service/backends/record_dir.py | 2 +- dump_things_service/backends/sqlite.py | 2 +- dump_things_service/utils.py | 60 +++++++++++++--------- 5 files changed, 43 insertions(+), 31 deletions(-) diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 1eba71b..e937efe 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -162,8 +162,8 @@ class AdminTokenConfig(StrictModel): class Configuration(StrictModel): - type: str = Literal['collections'] - version: int = Literal[2] + type: Literal['collections'] + version: Literal[2] collections: dict[str, CollectionConfig] = {} tokens: dict[str, TokenConfig] = {} admin_tokens: dict[str, AdminTokenConfig] = {} @@ -380,7 +380,7 @@ def get_token_infos_for_collection( yield from { (token_name, token_config, token_collection_config) for token_name, token_config in abstract_config.tokens.items() - for token_collection_config in token_config.collections.get(collection_name) + for token_collection_config in token_config.collections.get(collection_name, []) if token_config is not None } @@ -473,7 +473,7 @@ def mapping_after_last_colon(pid: str, suffix: str) -> Path: return Path(escaped_result + '.' + suffix) -mapping_functions = { +mapping_functions: dict[MappingMethod, Callable] = { MappingMethod.digest_md5: partial(mapping_digest, hashlib.md5), MappingMethod.digest_md5_p3: partial(mapping_digest_p3, hashlib.md5), MappingMethod.digest_md5_p3_p3: partial(mapping_digest_p3_p3, hashlib.md5), diff --git a/dump_things_service/auth/__init__.py b/dump_things_service/auth/__init__.py index 00ff138..5758cb5 100644 --- a/dump_things_service/auth/__init__.py +++ b/dump_things_service/auth/__init__.py @@ -31,7 +31,7 @@ class InvalidTokenError(AuthenticationError): class AuthenticationInfo: token_permission: TokenPermission user_id: str - incoming_label: str | None + incoming_label: str class AuthenticationSource(metaclass=abc.ABCMeta): diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 3fe67d5..533059f 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -218,7 +218,7 @@ class _RecordDirStore(StorageBackend): # Ensure that there is only one store per root directory. -_existing_stores = {} +_existing_stores: dict[Path, _RecordDirStore] = {} def RecordDirStore( # noqa: N802 diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index e0bc3df..0255c58 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -306,7 +306,7 @@ class _SQLiteBackend(StorageBackend): # Ensure that there is only one SQL-backend per database file. -_existing_sqlite_backends = {} +_existing_sqlite_backends: dict[Path, _SQLiteBackend] = {} def SQLiteBackend( # noqa: N802 diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index c2d9a9a..e184ca9 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -15,6 +15,7 @@ from contextlib import contextmanager from functools import reduce from typing import ( TYPE_CHECKING, + cast, ) import fsspec @@ -36,7 +37,7 @@ from dump_things_service.abstract_config import ( check_collection, get_collection_config_by_name, get_mapping_function_by_name, - mode_mapping, + mode_mapping, RecordDirBackendConfig, ) from dump_things_service.auth import ( AuthenticationError, @@ -90,12 +91,14 @@ def cleaned_json(data: JSON, remove_keys: tuple[str, ...] = ('@type',)) -> JSON: def combine_ttl(documents: list[str]) -> str: graphs = [Graph().parse(data=doc, format='ttl') for doc in documents] + if not graphs: + return '' return reduce(lambda g1, g2: g1 + g2, graphs).serialize(format='ttl') @contextmanager def wrap_http_exception( - exception_class: type[BaseException] = ValueError, + exception_class: type[Exception] = ValueError, status_code: int = HTTP_400_BAD_REQUEST, header: str = '', ): @@ -155,7 +158,10 @@ def get_on_disk_labels( ) -> set[str]: check_collection(abstract_config, collection) - incoming_path = store_path / abstract_config.collections[collection].incoming + if abstract_config.collections[collection].incoming is None: + return set() + + incoming_path = store_path / cast(Path, abstract_config.collections[collection].incoming) if not incoming_path or not incoming_path.exists(): return set() @@ -222,31 +228,26 @@ def get_token_store( abstract_config: Configuration, instance_state: InstanceState, collection_name: str, - token_representation: str | None, + token_representation: str, *, is_token_name: bool = False, -) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]: +) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None]: # If a token representation is provided, try to authenticate the token # with the authentication providers that are associated with the collection. if not is_token_name: - if token_representation is None: - msg = 'get_token_store: token_representation is None and is_token_name is False. This is a calling error!' - logger.error(msg) - raise ValueError(msg) - auth_info = authenticate_token( instance_state, collection_name, token_representation, ) - + token_key = token_representation else: auth_info = get_default_token_auth_info( abstract_config=abstract_config, collection_name=collection_name, token_name=token_representation, ) - token_representation = None + token_key = None if not auth_info: detail = f"invalid token for collection '{collection_name}'" @@ -259,12 +260,12 @@ def get_token_store( # If the token has no incoming-read or incoming-write permissions, we do not # need to create a store. if not permissions.incoming_read and not permissions.incoming_write: - instance_state.incoming_stores[collection_name][token_representation] = ( + instance_state.incoming_stores[collection_name][token_key] = ( None, permissions, auth_info.user_id, ) - return instance_state.incoming_stores[collection_name][token_representation] + return instance_state.incoming_stores[collection_name][token_key] # Check whether the collection has an incoming definition incoming = abstract_config.collections[collection_name].incoming @@ -275,9 +276,7 @@ def get_token_store( ) # Check whether a store for this collection and token does already exist. - store_info = instance_state.incoming_stores[collection_name].get( - token_representation - ) + store_info = instance_state.incoming_stores[collection_name].get(token_key) if store_info: return store_info @@ -289,12 +288,12 @@ def get_token_store( store_dir=store_dir, ) - instance_state.incoming_stores[collection_name][token_representation] = ( + instance_state.incoming_stores[collection_name][token_key] = ( token_store, permissions, auth_info.user_id, ) - return instance_state.incoming_stores[collection_name][token_representation] + return instance_state.incoming_stores[collection_name][token_key] def create_store( @@ -354,7 +353,7 @@ def create_token_store( store_dir=store_dir, order_by=instance_state.order_by, schema_uri=schema_uri, - mapping_function=backend_config.mapping_method, + mapping_function=cast(RecordDirBackendConfig, backend_config).mapping_method, suffix='yaml', ) elif backend_name == 'sqlite': @@ -450,7 +449,7 @@ def create_sqlite_token_store_backend( def check_bounds( length: int | None, max_length: int, collection: str, alternative_url: str ): - if length > max_length: + if length is not None and length > max_length: raise HTTPException( status_code=HTTP_413_CONTENT_TOO_LARGE, detail=f"Too many records found in collection '{collection}'. " @@ -478,7 +477,14 @@ async def process_token( abstract_config, instance_state, collection, - api_key, + token_representation=api_key, + is_token_name=False, + ) + + if token_store is None or token_permissions is None: + raise HTTPException( + status_code=HTTP_401_UNAUTHORIZED, + detail=f"No read access to curated or incoming data in collection '{collection}'.", ) final_permissions = join_default_token_permissions( @@ -531,10 +537,10 @@ def var_escape( def json2yaml( - json: dict, + json_object: dict, ) -> str: return yaml.dump( - data=json, + data=json_object, sort_keys=True, allow_unicode=True, default_flow_style=False, @@ -542,6 +548,12 @@ def json2yaml( def order_dict( + d: dict, +) -> dict: + return cast(dict, _order_dict(d)) + + +def _order_dict( d: dict | list | str | int | None, ) -> dict | list | str | int | None: if isinstance(d, dict): -- 2.52.0 From e2c58089e2bea8147a4683ea25e688fafa7e5e47 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Fri, 3 Jul 2026 15:45:42 +0200 Subject: [PATCH 8/9] chore: format with `hatch check format --fix` --- dump_things_service/utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index e184ca9..560ddc3 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -37,7 +37,8 @@ from dump_things_service.abstract_config import ( check_collection, get_collection_config_by_name, get_mapping_function_by_name, - mode_mapping, RecordDirBackendConfig, + mode_mapping, + RecordDirBackendConfig, ) from dump_things_service.auth import ( AuthenticationError, @@ -161,7 +162,9 @@ def get_on_disk_labels( if abstract_config.collections[collection].incoming is None: return set() - incoming_path = store_path / cast(Path, abstract_config.collections[collection].incoming) + incoming_path = store_path / cast( + Path, abstract_config.collections[collection].incoming + ) if not incoming_path or not incoming_path.exists(): return set() @@ -353,7 +356,9 @@ def create_token_store( store_dir=store_dir, order_by=instance_state.order_by, schema_uri=schema_uri, - mapping_function=cast(RecordDirBackendConfig, backend_config).mapping_method, + mapping_function=cast( + RecordDirBackendConfig, backend_config + ).mapping_method, suffix='yaml', ) elif backend_name == 'sqlite': -- 2.52.0 From 3005ab6052828f53caf7a40ea14a6952d31ce3ac Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Fri, 3 Jul 2026 15:57:08 +0200 Subject: [PATCH 9/9] fix: fix types and faulty authentication check --- dump_things_service/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index 560ddc3..2c50572 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -32,13 +32,13 @@ from dump_things_service import ( ) from dump_things_service.abstract_config import ( Configuration, + RecordDirBackendConfig, TokenModes, TokenPermission, check_collection, get_collection_config_by_name, get_mapping_function_by_name, mode_mapping, - RecordDirBackendConfig, ) from dump_things_service.auth import ( AuthenticationError, @@ -163,7 +163,7 @@ def get_on_disk_labels( return set() incoming_path = store_path / cast( - Path, abstract_config.collections[collection].incoming + 'Path', abstract_config.collections[collection].incoming ) if not incoming_path or not incoming_path.exists(): return set() @@ -357,7 +357,7 @@ def create_token_store( order_by=instance_state.order_by, schema_uri=schema_uri, mapping_function=cast( - RecordDirBackendConfig, backend_config + 'RecordDirBackendConfig', backend_config ).mapping_method, suffix='yaml', ) @@ -486,7 +486,7 @@ async def process_token( is_token_name=False, ) - if token_store is None or token_permissions is None: + if token_permissions is None: raise HTTPException( status_code=HTTP_401_UNAUTHORIZED, detail=f"No read access to curated or incoming data in collection '{collection}'.", @@ -555,7 +555,7 @@ def json2yaml( def order_dict( d: dict, ) -> dict: - return cast(dict, _order_dict(d)) + return cast('dict', _order_dict(d)) def _order_dict( -- 2.52.0