dump_things_service/abstract_config.py

@@ -162,8 +162,8 @@ class AdminTokenConfig(StrictModel):
 
 
 class Configuration(StrictModel):
-    type: str = Literal['collections']
-    version: int = Literal[2]
+    type: Literal['collections']
+    version: Literal[2]
     collections: dict[str, CollectionConfig] = {}
     tokens: dict[str, TokenConfig] = {}
     admin_tokens: dict[str, AdminTokenConfig] = {}
@@ -267,18 +267,21 @@ def store_config(
     store_path,
     config: Configuration,
 ):
+    from dump_things_service.utils import order_dict  # noqa PLC0415 -- global import leads to circular imports
+
     global g_abstract_configuration  # noqa PLW0603 -- this function updates a globally referenced instance
 
     config_backend, audit_backend = get_config_backends(store_path)
     json_object = config.model_dump(mode='json', exclude_none=True, by_alias=True)
     json_object['pid'] = dump_things_config_iri
+    sorted_json_object = order_dict(json_object)
     config_backend.add_record(
         iri=dump_things_config_iri,
         class_name='DumpThingsConfig',
-        json_object=json_object,
+        json_object=sorted_json_object,
     )
     audit_backend.add_record(
-        record=json_object,
+        record=sorted_json_object,
         committer_id='__dump_things_server__',
     )
     g_abstract_configuration = config
@@ -377,7 +380,7 @@ def get_token_infos_for_collection(
     yield from {
         (token_name, token_config, token_collection_config)
         for token_name, token_config in abstract_config.tokens.items()
-        for token_collection_config in token_config.collections.get(collection_name)
+        for token_collection_config in token_config.collections.get(collection_name, [])
         if token_config is not None
     }
 
@@ -470,7 +473,7 @@ def mapping_after_last_colon(pid: str, suffix: str) -> Path:
     return Path(escaped_result + '.' + suffix)
 
 
-mapping_functions = {
+mapping_functions: dict[MappingMethod, Callable] = {
     MappingMethod.digest_md5: partial(mapping_digest, hashlib.md5),
     MappingMethod.digest_md5_p3: partial(mapping_digest_p3, hashlib.md5),
     MappingMethod.digest_md5_p3_p3: partial(mapping_digest_p3_p3, hashlib.md5),

dump_things_service/audit/gitaudit.py

@@ -223,14 +223,11 @@ class GitAuditBackend(AuditBackend):
         author_id: str,
         record: dict,
     ) -> bool:
+        from dump_things_service.utils import json2yaml  # noqa PLC0415 -- global import leads to circular imports
+
         existing_record = self._read_record_from_repo_path(location[1])
         if existing_record != record:
-            self.current_change_set[location[1]] = yaml.dump(
-                data=record,
-                sort_keys=False,
-                allow_unicode=True,
-                default_flow_style=False,
-            )
+            self.current_change_set[location[1]] = json2yaml(record)
             self._add_log_entry(location[2], committer_id, author_id)
             self._add_index_entry(record_id)
             return True
@@ -249,7 +246,7 @@ class GitAuditBackend(AuditBackend):
             'author_id': author_id,
         }
         log_content = self._read_from_repo_path(log_location).decode()
-        log_content += json.dumps(entry, ensure_ascii=False) + '\n'
+        log_content += json.dumps(entry, ensure_ascii=False, sort_keys=True) + '\n'
         self.current_change_set[log_location] = log_content
 
     def _add_index_entry(

dump_things_service/auth/__init__.py

@@ -31,7 +31,7 @@ class InvalidTokenError(AuthenticationError):
 class AuthenticationInfo:
     token_permission: TokenPermission
     user_id: str
-    incoming_label: str | None
+    incoming_label: str
 
 
 class AuthenticationSource(metaclass=abc.ABCMeta):

dump_things_service/backends/record_dir.py

@@ -61,12 +61,14 @@ class RecordDirResultList(BackendResultList):
         :param path: The path where the record is stored
         :return: A RecordInfo object.
         """
+        from dump_things_service.utils import order_dict  # noqa PLC0415 -- global import leads to circular imports
+
         with path.open('r') as f:
             json_object = yaml.load(f, Loader=yaml.SafeLoader)
             return RecordInfo(
                 iri=iri,
                 class_name=class_name,
-                json_object=json_object,
+                json_object=order_dict(json_object),
                 sort_key=sort_key,
             )
 
@@ -111,6 +113,8 @@ class _RecordDirStore(StorageBackend):
         class_name: str,
         json_object: dict,
     ):
+        from dump_things_service.utils import json2yaml  # noqa PLC0415 -- global import leads to circular imports
+
         pid = json_object['pid']
 
         # Generate the class directory, apply the mapping function to the record
@@ -132,12 +136,7 @@ class _RecordDirStore(StorageBackend):
         storage_path.parent.mkdir(parents=True, exist_ok=True)
 
         # Convert the record object into a YAML object
-        data = yaml.dump(
-            data=json_object,
-            sort_keys=False,
-            allow_unicode=True,
-            default_flow_style=False,
-        )
+        data = json2yaml(json_object)
         storage_path.write_text(data, encoding='utf-8')
 
         # Add the IRI to the index.
@@ -219,7 +218,7 @@ class _RecordDirStore(StorageBackend):
 
 
 # Ensure that there is only one store per root directory.
-_existing_stores = {}
+_existing_stores: dict[Path, _RecordDirStore] = {}
 
 
 def RecordDirStore(  # noqa: N802

dump_things_service/backends/record_dir_index.py

@@ -41,6 +41,7 @@ if TYPE_CHECKING:
 __all__ = [
     'IndexEntry',
     'RecordDirIndex',
+    'index_file_name',
 ]
 
 index_file_name = '.directory_dir_index.db'

dump_things_service/backends/sqlite.py

@@ -64,6 +64,7 @@ if TYPE_CHECKING:
 __all__ = [
     'SQLiteBackend',
     '_SQLiteBackend',
+    'record_file_name',
 ]
 
 logger = logging.getLogger('dump_things_service')
@@ -305,7 +306,7 @@ class _SQLiteBackend(StorageBackend):
 
 
 # Ensure that there is only one SQL-backend per database file.
-_existing_sqlite_backends = {}
+_existing_sqlite_backends: dict[Path, _SQLiteBackend] = {}
 
 
 def SQLiteBackend(  # noqa: N802

dump_things_service/collection.py

@@ -80,6 +80,7 @@ from dump_things_service.utils import (
     create_store,
     get_token_store,
     join_default_token_permissions,
+    order_dict,
     var_escape,
     wrap_http_exception,
 )
@@ -665,9 +666,10 @@ def store_record(
     ):
         instance_state.validators[collection].validate(record)
 
+    sorted_record = order_dict(record)
     with wrap_http_exception(CurieResolutionError):
         stored_records = store.store_object(
-            obj=record,
+            obj=sorted_record,
             submitter=user_id if add_submission_tag else None,
         )
 
@@ -690,4 +692,4 @@ def store_record(
                 ),
                 media_type='text/turtle',
             )
-    return JSONResponse([record for _, record in stored_records])
+    return JSONResponse([order_dict(record) for _, record in stored_records])

dump_things_service/commands/download_config.py

@@ -6,9 +6,10 @@ import sys
 from argparse import ArgumentParser
 
 import requests
-import yaml
 from starlette.status import HTTP_300_MULTIPLE_CHOICES
 
+from dump_things_service.utils import json2yaml
+
 parser = ArgumentParser(
     prog='Download a complete configuration of a running service',
     description='Read a configuration from dump-things endpoints and create a '
@@ -76,14 +77,7 @@ def main():
     if arguments.format == 'json':
         print(json.dumps(configuration, indent=2, sort_keys=False))  # noqa T201 -- cli result output
     elif arguments.format == 'yaml':
-        print(  # noqa T201 -- cli result output
-            yaml.dump(
-                data=configuration,
-                sort_keys=False,
-                allow_unicode=True,
-                default_flow_style=False,
-            )
-        )
+        print(json2yaml(configuration))  # noqa T201 -- cli result output
     return 0
 
 

dump_things_service/commands/upload_config.py

@@ -12,6 +12,7 @@ import yaml
 from starlette.status import HTTP_300_MULTIPLE_CHOICES
 
 from dump_things_service.instance_state import get_record_dir_config
+from dump_things_service.utils import json2yaml
 
 CONFIG_VERSION = 2
 
@@ -129,14 +130,7 @@ def main():
     if file_type == 'json':
         print(json.dumps(configuration, indent=2, sort_keys=False))  # noqa T201 -- cli result output
     elif file_type == 'yaml':
-        print(  # noqa T201 -- cli result output
-            yaml.dump(
-                data=configuration,
-                sort_keys=False,
-                allow_unicode=True,
-                default_flow_style=False,
-            )
-        )
+        print(json2yaml(configuration))  # noqa T201 -- cli result output
     return 0
 
 

dump_things_service/curated.py

@@ -13,6 +13,7 @@ from fastapi_pagination import (
     add_pagination,
     paginate,
 )
+from starlette.status import HTTP_403_FORBIDDEN
 
 from dump_things_service import (
     HTTP_401_UNAUTHORIZED,
@@ -26,12 +27,16 @@ from dump_things_service.abstract_config import (
 from dump_things_service.api_key import api_key_header_scheme
 from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer
 from dump_things_service.exceptions import CurieResolutionError
-from dump_things_service.instance_state import get_instance_state
+from dump_things_service.instance_state import (
+    check_instance_state_collection,
+    get_instance_state,
+)
 from dump_things_service.lazy_list import ModifierList
 from dump_things_service.utils import (
     authenticate_token,
     check_bounds,
     cleaned_json,
+    order_dict,
     wrap_http_exception,
 )
 
@@ -56,7 +61,7 @@ async def {name}(
         repr(author_id),
         repr({model_var_name}),
     )
-    return await store_curated_record(
+    return store_curated_record(
         '{collection}',
         data,
         '{class_name}',
@@ -83,6 +88,7 @@ async def read_curated_records_of_type(
     matching: str | None = None,
 ):
     instance_state = get_instance_state()
+    check_instance_state_collection(instance_state, collection)
     if class_name not in instance_state.collections[collection].active_classes:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
@@ -111,6 +117,7 @@ async def read_curated_records_of_type_paginated(
     matching: str | None = None,
 ) -> Page[dict]:
     instance_state = get_instance_state()
+    check_instance_state_collection(instance_state, collection)
     if class_name not in instance_state.collections[collection].active_classes:
         raise HTTPException(
             status_code=HTTP_404_NOT_FOUND,
@@ -218,6 +225,13 @@ async def _read_curated_records(
         if record_info:
             return record_info.json_object
         return None
+        # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of
+        #  returning an empty HTTP_200_OK result. This will influence on how the
+        #  endpoint is used by the clients.
+        # raise HTTPException(
+        #    status_code=HTTP_404_NOT_FOUND,
+        #    detail=f"no record with pid '{pid}' in curated area of collection '{collection}'",
+        # )
     if class_name:
         result_list = backend.get_records_of_classes([class_name], matching)
     else:
@@ -276,7 +290,7 @@ def _get_store_and_backend(
     permissions = auth_info.token_permission
     if permissions.curated_write is False:
         raise HTTPException(
-            status_code=HTTP_401_UNAUTHORIZED,
+            status_code=HTTP_403_FORBIDDEN,
             detail=f'no write access to curated area of collection `{collection}`',
         )
 
@@ -292,8 +306,8 @@ def store_curated_record(
     collection: str,
     data: BaseModel,
     class_name: str,
-    author_id: str | None = None,
-    api_key: str | None = Depends(api_key_header_scheme),
+    author_id: str | None,
+    api_key: str | None,
 ):
     instance_state = get_instance_state()
     with wrap_http_exception(
@@ -311,16 +325,17 @@ def store_curated_record(
         remove_keys=('@type',),
     )
 
+    sorted_json_object = order_dict(json_object)
     with wrap_http_exception(CurieResolutionError):
         backend.add_record(
             model_store.pid_to_iri(pid),
             class_name,
-            json_object,
+            sorted_json_object,
         )
 
     for audit_backend in instance_state.audit_backends[collection]:
         audit_backend.add_record(
-            record=json_object,
+            record=sorted_json_object,
             committer_id=auth_info.user_id,
             author_id=author_id,
         )

dump_things_service/incoming.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Annotated
 
 from fastapi import (
@@ -13,6 +12,7 @@ from fastapi_pagination import (
     add_pagination,
     paginate,
 )
+from starlette.status import HTTP_403_FORBIDDEN
 
 from dump_things_service import (
     HTTP_401_UNAUTHORIZED,
@@ -28,7 +28,11 @@ from dump_things_service.abstract_config import (
 from dump_things_service.api_key import api_key_header_scheme
 from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer
 from dump_things_service.exceptions import CurieResolutionError
-from dump_things_service.instance_state import get_instance_state
+from dump_things_service.instance_state import (
+    InstanceState,
+    check_instance_state_collection,
+    get_instance_state,
+)
 from dump_things_service.lazy_list import ModifierList
 from dump_things_service.utils import (
     authenticate_token,
@@ -36,6 +40,7 @@ from dump_things_service.utils import (
     cleaned_json,
     create_token_store,
     get_on_disk_labels,
+    order_dict,
     wrap_http_exception,
 )
 
@@ -47,7 +52,6 @@ if TYPE_CHECKING:
     from dump_things_service.store.model_store import _ModelStore
 
 
-logger = logging.getLogger('dump_things_service')
 router = APIRouter()
 add_pagination(router)
 
@@ -69,7 +73,7 @@ async def incoming_read_labels(
     on_disk_labels = get_on_disk_labels(
         instance_state.store_path, get_config(), collection
     )
-    return list(configured_labels.union(on_disk_labels))
+    return sorted(configured_labels.union(on_disk_labels))
 
 
 @router.get(
@@ -85,11 +89,7 @@ async def incoming_read_records_of_type(
     matching: str | None = None,
 ):
     instance_state = get_instance_state()
-    if class_name not in instance_state.collections[collection].active_classes:
-        raise HTTPException(
-            status_code=HTTP_404_NOT_FOUND,
-            detail=f"No '{class_name}'-class in collection '{collection}'.",
-        )
+    _check_collection_and_class(instance_state, collection, class_name)
 
     return await _incoming_read_records(
         collection=collection,
@@ -115,11 +115,7 @@ async def incoming_read_records_of_type_paginated(
     matching: str | None = None,
 ) -> Page[dict]:
     instance_state = get_instance_state()
-    if class_name not in instance_state.collections[collection].active_classes:
-        raise HTTPException(
-            status_code=HTTP_404_NOT_FOUND,
-            detail=f"No '{class_name}'-class in collection '{collection}'.",
-        )
+    _check_collection_and_class(instance_state, collection, class_name)
 
     record_list = await _incoming_read_records(
         collection=collection,
@@ -128,10 +124,24 @@ async def incoming_read_records_of_type_paginated(
         pid=None,
         matching=matching,
         api_key=api_key,
+        upper_bound=None,
     )
     return paginate(record_list)
 
 
+def _check_collection_and_class(
+    instance_state: InstanceState,
+    collection: str,
+    class_name: str,
+):
+    check_instance_state_collection(instance_state, collection)
+    if class_name not in instance_state.collections[collection].active_classes:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No '{class_name}'-class in collection '{collection}'.",
+        )
+
+
 @router.get(
     '/{collection}/incoming/{label}/records/',
     tags=['Incoming area: read records'],
@@ -223,7 +233,7 @@ async def _incoming_read_records(
     pid: str | None,
     matching: str | None = None,
     api_key: str | None = None,
-    upper_bound: int = 1000,
+    upper_bound: int | None = 1000,
 ) -> LazyList | dict | None:
     model_store, backend = await _get_store_and_backend(collection, label, api_key)
 
@@ -232,6 +242,13 @@ async def _incoming_read_records(
         if record_info:
             return record_info.json_object
         return None
+        # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of
+        #  returning an empty HTTP_200_OK result. This will influence on how the
+        #  endpoint is used by the clients.
+        # raise HTTPException(
+        #    status_code=HTTP_404_NOT_FOUND,
+        #    detail=f'no record with pid: {pid}',
+        # )
     if class_name:
         result_list = backend.get_records_of_classes([class_name], matching)
     else:
@@ -242,9 +259,9 @@ async def _incoming_read_records(
             len(result_list),
             upper_bound,
             collection,
-            f'/incoming/{label}/records/p/{class_name}'
+            f'/incoming/{label}/records/{class_name}'
             if class_name
-            else f'/incoming/{label}/records/p/',
+            else f'/incoming/{label}/records/',
         )
 
     return ModifierList(
@@ -256,7 +273,7 @@ async def _incoming_read_records(
 async def _incoming_delete_record(
     collection: str,
     label: str,
-    pid: str | None,
+    pid: str,
     api_key: str | None = None,
 ) -> bool:
     model_store, backend = await _get_store_and_backend(collection, label, api_key)
@@ -274,10 +291,10 @@ async def _incoming_delete_record(
 async def _get_store_and_backend(
     collection: str,
     label: str,
-    plain_token: str | None,
+    api_key: str | None,
 ) -> tuple[_ModelStore, StorageBackend]:
     # Authorize api_key
-    await authorize_zones(collection, plain_token)
+    await authorize_zones(collection, api_key)
 
     # Check that the incoming zone exists
     instance_state = get_instance_state()
@@ -327,7 +344,7 @@ async def authorize_zones(
     permissions = auth_info.token_permission
     if permissions.zones_access is False:
         raise HTTPException(
-            status_code=HTTP_401_UNAUTHORIZED,
+            status_code=HTTP_403_FORBIDDEN,
             detail=f'no access to incoming zones of collection `{collection}`',
         )
 
@@ -337,7 +354,7 @@ async def store_incoming_record(
     label: str,
     data: BaseModel,
     class_name: str,
-    api_key: str | None = Depends(api_key_header_scheme),
+    api_key: str | None,
 ):
     instance_state = get_instance_state()
     with wrap_http_exception(
@@ -359,9 +376,10 @@ async def store_incoming_record(
         remove_keys=('@type',),
     )
 
+    sorted_json_object = order_dict(json_object)
     with wrap_http_exception(CurieResolutionError):
         backend.add_record(
             model_store.pid_to_iri(pid),
             class_name,
-            json_object,
+            sorted_json_object,
         )

dump_things_service/instance_state.py

@@ -9,9 +9,11 @@ from typing import (
 )
 
 import yaml
+from fastapi import HTTPException
 from pydantic import ValidationError
 from yaml.scanner import ScannerError
 
+from dump_things_service import HTTP_404_NOT_FOUND
 from dump_things_service.abstract_config import (
     MappingMethod,
     RecordDirConfigFileContent,
@@ -155,3 +157,14 @@ def get_mapping_function_by_name(mapping_function_name: str) -> Callable:
 
 def get_mapping_function(collection_config: RecordDirConfigFileContent):
     return mapping_functions[collection_config.idfx]
+
+
+def check_instance_state_collection(
+    instance_state: InstanceState,
+    collection: str,
+):
+    if collection not in instance_state.collections:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No such collection: '{collection}'.",
+        )

dump_things_service/tests/fixtures.py

@@ -395,6 +395,7 @@ def fastapi_client_simple(fastapi_app_simple):
         response = test_client.post(
             '/collections',
             json=collection_config.model_dump(
+                exclude_none=True,
                 exclude_unset=True,
                 mode='json',
                 by_alias=True,
@@ -407,7 +408,11 @@ def fastapi_client_simple(fastapi_app_simple):
     for token_config in g_default_tokens:
         response = test_client.post(
             '/tokens',
-            json=token_config.model_dump(exclude_unset=True, mode='json'),
+            json=token_config.model_dump(
+                exclude_none=True,
+                exclude_unset=True,
+                mode='json',
+            ),
             headers={'x-dumpthings-token': admin_token},
         )
         assert response.status_code == 201

dump_things_service/tests/test_canonical.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from dump_things_service import HTTP_200_OK
+from dump_things_service.abstract_config import (
+    MappingMethod,
+    mapping_functions,
+)
+
+
+def test_canonicalization(fastapi_client_simple):
+    test_client, store_path, _admin_token = fastapi_client_simple
+
+    pid = 'http://example.com/test_canonicalization/1'
+    record_a = {
+        'pid': pid,
+        'given_name': 'Alice',
+    }
+
+    record_b = {
+        'given_name': 'Bob',
+        'pid': pid,
+    }
+
+    # Ensure that the two records are represented in the same order on disk.
+    # (The current implementation should guarantee that because the pydantic
+    # objects that are instantiated from the JSON input have a fixed order
+    # of attributes. Once the implementation is changed, this test here might
+    # catch regressions).
+    file_content = []
+    for record in (record_a, record_b):
+        response = test_client.post(
+            '/collection_1/record/Person',
+            headers={'x-dumpthings-token': 'token-1'},
+            json=record,
+        )
+        assert response.status_code == HTTP_200_OK
+        file_content.append(
+            _get_record_dir_record(
+                store_path,
+                'incoming/collection_1/in_token_1/Person',
+                pid,
+                MappingMethod.digest_md5,
+            ),
+        )
+    assert file_content[0] == file_content[1].replace('Bob', 'Alice')
+
+
+def _get_record_dir_record(
+    store_path: Path,
+    path_part: str | Path,
+    pid: str,
+    mapping_method: MappingMethod,
+) -> str:
+    file_name = mapping_functions[mapping_method](pid, 'yaml')
+    with (store_path / path_part / file_name).open('rt') as f:
+        return f.read()

dump_things_service/utils.py

@@ -15,9 +15,11 @@ from contextlib import contextmanager
 from functools import reduce
 from typing import (
     TYPE_CHECKING,
+    cast,
 )
 
 import fsspec
+import yaml
 from fastapi import HTTPException
 from rdflib import Graph
 
@@ -30,6 +32,7 @@ from dump_things_service import (
 )
 from dump_things_service.abstract_config import (
     Configuration,
+    RecordDirBackendConfig,
     TokenModes,
     TokenPermission,
     check_collection,
@@ -89,12 +92,14 @@ def cleaned_json(data: JSON, remove_keys: tuple[str, ...] = ('@type',)) -> JSON:
 
 def combine_ttl(documents: list[str]) -> str:
     graphs = [Graph().parse(data=doc, format='ttl') for doc in documents]
+    if not graphs:
+        return ''
     return reduce(lambda g1, g2: g1 + g2, graphs).serialize(format='ttl')
 
 
 @contextmanager
 def wrap_http_exception(
-    exception_class: type[BaseException] = ValueError,
+    exception_class: type[Exception] = ValueError,
     status_code: int = HTTP_400_BAD_REQUEST,
     header: str = '',
 ):
@@ -154,7 +159,12 @@ def get_on_disk_labels(
 ) -> set[str]:
     check_collection(abstract_config, collection)
 
-    incoming_path = store_path / abstract_config.collections[collection].incoming
+    if abstract_config.collections[collection].incoming is None:
+        return set()
+
+    incoming_path = store_path / cast(
+        'Path', abstract_config.collections[collection].incoming
+    )
     if not incoming_path or not incoming_path.exists():
         return set()
 
@@ -221,31 +231,26 @@ def get_token_store(
     abstract_config: Configuration,
     instance_state: InstanceState,
     collection_name: str,
-    token_representation: str | None,
+    token_representation: str,
     *,
     is_token_name: bool = False,
-) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]:
+) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None]:
     # If a token representation is provided, try to authenticate the token
     # with the authentication providers that are associated with the collection.
     if not is_token_name:
-        if token_representation is None:
-            msg = 'get_token_store: token_representation is None and is_token_name is False. This is a calling error!'
-            logger.error(msg)
-            raise ValueError(msg)
-
         auth_info = authenticate_token(
             instance_state,
             collection_name,
             token_representation,
         )
-
+        token_key = token_representation
     else:
         auth_info = get_default_token_auth_info(
             abstract_config=abstract_config,
             collection_name=collection_name,
             token_name=token_representation,
         )
-        token_representation = None
+        token_key = None
 
     if not auth_info:
         detail = f"invalid token for collection '{collection_name}'"
@@ -258,12 +263,12 @@ def get_token_store(
     # If the token has no incoming-read or incoming-write permissions, we do not
     # need to create a store.
     if not permissions.incoming_read and not permissions.incoming_write:
-        instance_state.incoming_stores[collection_name][token_representation] = (
+        instance_state.incoming_stores[collection_name][token_key] = (
             None,
             permissions,
             auth_info.user_id,
         )
-        return instance_state.incoming_stores[collection_name][token_representation]
+        return instance_state.incoming_stores[collection_name][token_key]
 
     # Check whether the collection has an incoming definition
     incoming = abstract_config.collections[collection_name].incoming
@@ -274,9 +279,7 @@ def get_token_store(
         )
 
     # Check whether a store for this collection and token does already exist.
-    store_info = instance_state.incoming_stores[collection_name].get(
-        token_representation
-    )
+    store_info = instance_state.incoming_stores[collection_name].get(token_key)
     if store_info:
         return store_info
 
@@ -288,12 +291,12 @@ def get_token_store(
         store_dir=store_dir,
     )
 
-    instance_state.incoming_stores[collection_name][token_representation] = (
+    instance_state.incoming_stores[collection_name][token_key] = (
         token_store,
         permissions,
         auth_info.user_id,
     )
-    return instance_state.incoming_stores[collection_name][token_representation]
+    return instance_state.incoming_stores[collection_name][token_key]
 
 
 def create_store(
@@ -353,7 +356,9 @@ def create_token_store(
             store_dir=store_dir,
             order_by=instance_state.order_by,
             schema_uri=schema_uri,
-            mapping_function=backend_config.mapping_method,
+            mapping_function=cast(
+                'RecordDirBackendConfig', backend_config
+            ).mapping_method,
             suffix='yaml',
         )
     elif backend_name == 'sqlite':
@@ -449,7 +454,7 @@ def create_sqlite_token_store_backend(
 def check_bounds(
     length: int | None, max_length: int, collection: str, alternative_url: str
 ):
-    if length > max_length:
+    if length is not None and length > max_length:
         raise HTTPException(
             status_code=HTTP_413_CONTENT_TOO_LARGE,
             detail=f"Too many records found in collection '{collection}'. "
@@ -477,7 +482,14 @@ async def process_token(
             abstract_config,
             instance_state,
             collection,
-            api_key,
+            token_representation=api_key,
+            is_token_name=False,
+        )
+
+    if token_permissions is None:
+        raise HTTPException(
+            status_code=HTTP_401_UNAUTHORIZED,
+            detail=f"No read access to curated or incoming data in collection '{collection}'.",
         )
 
     final_permissions = join_default_token_permissions(
@@ -527,3 +539,30 @@ def var_escape(
     name: str,
 ) -> str:
     return name.replace('_', '___').replace('-', '_0_')
+
+
+def json2yaml(
+    json_object: dict,
+) -> str:
+    return yaml.dump(
+        data=json_object,
+        sort_keys=True,
+        allow_unicode=True,
+        default_flow_style=False,
+    )
+
+
+def order_dict(
+    d: dict,
+) -> dict:
+    return cast('dict', _order_dict(d))
+
+
+def _order_dict(
+    d: dict | list | str | int | None,
+) -> dict | list | str | int | None:
+    if isinstance(d, dict):
+        return {k: order_dict(d[k]) for k in sorted(d)}
+    if isinstance(d, list):
+        return [order_dict(e) for e in d]
+    return d