diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 6ec9f16..e937efe 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -162,8 +162,8 @@ class AdminTokenConfig(StrictModel): class Configuration(StrictModel): - type: str = Literal['collections'] - version: int = Literal[2] + type: Literal['collections'] + version: Literal[2] collections: dict[str, CollectionConfig] = {} tokens: dict[str, TokenConfig] = {} admin_tokens: dict[str, AdminTokenConfig] = {} @@ -267,18 +267,21 @@ def store_config( store_path, config: Configuration, ): + from dump_things_service.utils import order_dict # noqa PLC0415 -- global import leads to circular imports + global g_abstract_configuration # noqa PLW0603 -- this function updates a globally referenced instance config_backend, audit_backend = get_config_backends(store_path) json_object = config.model_dump(mode='json', exclude_none=True, by_alias=True) json_object['pid'] = dump_things_config_iri + sorted_json_object = order_dict(json_object) config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object=json_object, + json_object=sorted_json_object, ) audit_backend.add_record( - record=json_object, + record=sorted_json_object, committer_id='__dump_things_server__', ) g_abstract_configuration = config @@ -377,7 +380,7 @@ def get_token_infos_for_collection( yield from { (token_name, token_config, token_collection_config) for token_name, token_config in abstract_config.tokens.items() - for token_collection_config in token_config.collections.get(collection_name) + for token_collection_config in token_config.collections.get(collection_name, []) if token_config is not None } @@ -470,7 +473,7 @@ def mapping_after_last_colon(pid: str, suffix: str) -> Path: return Path(escaped_result + '.' + suffix) -mapping_functions = { +mapping_functions: dict[MappingMethod, Callable] = { MappingMethod.digest_md5: partial(mapping_digest, hashlib.md5), MappingMethod.digest_md5_p3: partial(mapping_digest_p3, hashlib.md5), MappingMethod.digest_md5_p3_p3: partial(mapping_digest_p3_p3, hashlib.md5), diff --git a/dump_things_service/audit/gitaudit.py b/dump_things_service/audit/gitaudit.py index 926babc..382b05d 100644 --- a/dump_things_service/audit/gitaudit.py +++ b/dump_things_service/audit/gitaudit.py @@ -223,14 +223,11 @@ class GitAuditBackend(AuditBackend): author_id: str, record: dict, ) -> bool: + from dump_things_service.utils import json2yaml # noqa PLC0415 -- global import leads to circular imports + existing_record = self._read_record_from_repo_path(location[1]) if existing_record != record: - self.current_change_set[location[1]] = yaml.dump( - data=record, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) + self.current_change_set[location[1]] = json2yaml(record) self._add_log_entry(location[2], committer_id, author_id) self._add_index_entry(record_id) return True @@ -249,7 +246,7 @@ class GitAuditBackend(AuditBackend): 'author_id': author_id, } log_content = self._read_from_repo_path(log_location).decode() - log_content += json.dumps(entry, ensure_ascii=False) + '\n' + log_content += json.dumps(entry, ensure_ascii=False, sort_keys=True) + '\n' self.current_change_set[log_location] = log_content def _add_index_entry( diff --git a/dump_things_service/auth/__init__.py b/dump_things_service/auth/__init__.py index 00ff138..5758cb5 100644 --- a/dump_things_service/auth/__init__.py +++ b/dump_things_service/auth/__init__.py @@ -31,7 +31,7 @@ class InvalidTokenError(AuthenticationError): class AuthenticationInfo: token_permission: TokenPermission user_id: str - incoming_label: str | None + incoming_label: str class AuthenticationSource(metaclass=abc.ABCMeta): diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 1af59db..533059f 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -61,12 +61,14 @@ class RecordDirResultList(BackendResultList): :param path: The path where the record is stored :return: A RecordInfo object. """ + from dump_things_service.utils import order_dict # noqa PLC0415 -- global import leads to circular imports + with path.open('r') as f: json_object = yaml.load(f, Loader=yaml.SafeLoader) return RecordInfo( iri=iri, class_name=class_name, - json_object=json_object, + json_object=order_dict(json_object), sort_key=sort_key, ) @@ -111,6 +113,8 @@ class _RecordDirStore(StorageBackend): class_name: str, json_object: dict, ): + from dump_things_service.utils import json2yaml # noqa PLC0415 -- global import leads to circular imports + pid = json_object['pid'] # Generate the class directory, apply the mapping function to the record @@ -132,12 +136,7 @@ class _RecordDirStore(StorageBackend): storage_path.parent.mkdir(parents=True, exist_ok=True) # Convert the record object into a YAML object - data = yaml.dump( - data=json_object, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) + data = json2yaml(json_object) storage_path.write_text(data, encoding='utf-8') # Add the IRI to the index. @@ -219,7 +218,7 @@ class _RecordDirStore(StorageBackend): # Ensure that there is only one store per root directory. -_existing_stores = {} +_existing_stores: dict[Path, _RecordDirStore] = {} def RecordDirStore( # noqa: N802 diff --git a/dump_things_service/backends/record_dir_index.py b/dump_things_service/backends/record_dir_index.py index 510d49d..723ee72 100644 --- a/dump_things_service/backends/record_dir_index.py +++ b/dump_things_service/backends/record_dir_index.py @@ -41,6 +41,7 @@ if TYPE_CHECKING: __all__ = [ 'IndexEntry', 'RecordDirIndex', + 'index_file_name', ] index_file_name = '.directory_dir_index.db' diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index 31049c2..0255c58 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -64,6 +64,7 @@ if TYPE_CHECKING: __all__ = [ 'SQLiteBackend', '_SQLiteBackend', + 'record_file_name', ] logger = logging.getLogger('dump_things_service') @@ -305,7 +306,7 @@ class _SQLiteBackend(StorageBackend): # Ensure that there is only one SQL-backend per database file. -_existing_sqlite_backends = {} +_existing_sqlite_backends: dict[Path, _SQLiteBackend] = {} def SQLiteBackend( # noqa: N802 diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 6be11ab..6ecb02e 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -80,6 +80,7 @@ from dump_things_service.utils import ( create_store, get_token_store, join_default_token_permissions, + order_dict, var_escape, wrap_http_exception, ) @@ -665,9 +666,10 @@ def store_record( ): instance_state.validators[collection].validate(record) + sorted_record = order_dict(record) with wrap_http_exception(CurieResolutionError): stored_records = store.store_object( - obj=record, + obj=sorted_record, submitter=user_id if add_submission_tag else None, ) @@ -690,4 +692,4 @@ def store_record( ), media_type='text/turtle', ) - return JSONResponse([record for _, record in stored_records]) + return JSONResponse([order_dict(record) for _, record in stored_records]) diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py index 3517535..65226c1 100644 --- a/dump_things_service/commands/download_config.py +++ b/dump_things_service/commands/download_config.py @@ -6,9 +6,10 @@ import sys from argparse import ArgumentParser import requests -import yaml from starlette.status import HTTP_300_MULTIPLE_CHOICES +from dump_things_service.utils import json2yaml + parser = ArgumentParser( prog='Download a complete configuration of a running service', description='Read a configuration from dump-things endpoints and create a ' @@ -76,14 +77,7 @@ def main(): if arguments.format == 'json': print(json.dumps(configuration, indent=2, sort_keys=False)) # noqa T201 -- cli result output elif arguments.format == 'yaml': - print( # noqa T201 -- cli result output - yaml.dump( - data=configuration, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) - ) + print(json2yaml(configuration)) # noqa T201 -- cli result output return 0 diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index f776e3f..cdd4abb 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -12,6 +12,7 @@ import yaml from starlette.status import HTTP_300_MULTIPLE_CHOICES from dump_things_service.instance_state import get_record_dir_config +from dump_things_service.utils import json2yaml CONFIG_VERSION = 2 @@ -129,14 +130,7 @@ def main(): if file_type == 'json': print(json.dumps(configuration, indent=2, sort_keys=False)) # noqa T201 -- cli result output elif file_type == 'yaml': - print( # noqa T201 -- cli result output - yaml.dump( - data=configuration, - sort_keys=False, - allow_unicode=True, - default_flow_style=False, - ) - ) + print(json2yaml(configuration)) # noqa T201 -- cli result output return 0 diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index bfb6976..d2c5b43 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -13,6 +13,7 @@ from fastapi_pagination import ( add_pagination, paginate, ) +from starlette.status import HTTP_403_FORBIDDEN from dump_things_service import ( HTTP_401_UNAUTHORIZED, @@ -26,12 +27,16 @@ from dump_things_service.abstract_config import ( from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError -from dump_things_service.instance_state import get_instance_state +from dump_things_service.instance_state import ( + check_instance_state_collection, + get_instance_state, +) from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, check_bounds, cleaned_json, + order_dict, wrap_http_exception, ) @@ -56,7 +61,7 @@ async def {name}( repr(author_id), repr({model_var_name}), ) - return await store_curated_record( + return store_curated_record( '{collection}', data, '{class_name}', @@ -83,6 +88,7 @@ async def read_curated_records_of_type( matching: str | None = None, ): instance_state = get_instance_state() + check_instance_state_collection(instance_state, collection) if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( status_code=HTTP_404_NOT_FOUND, @@ -111,6 +117,7 @@ async def read_curated_records_of_type_paginated( matching: str | None = None, ) -> Page[dict]: instance_state = get_instance_state() + check_instance_state_collection(instance_state, collection) if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( status_code=HTTP_404_NOT_FOUND, @@ -218,6 +225,13 @@ async def _read_curated_records( if record_info: return record_info.json_object return None + # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of + # returning an empty HTTP_200_OK result. This will influence on how the + # endpoint is used by the clients. + # raise HTTPException( + # status_code=HTTP_404_NOT_FOUND, + # detail=f"no record with pid '{pid}' in curated area of collection '{collection}'", + # ) if class_name: result_list = backend.get_records_of_classes([class_name], matching) else: @@ -276,7 +290,7 @@ def _get_store_and_backend( permissions = auth_info.token_permission if permissions.curated_write is False: raise HTTPException( - status_code=HTTP_401_UNAUTHORIZED, + status_code=HTTP_403_FORBIDDEN, detail=f'no write access to curated area of collection `{collection}`', ) @@ -292,8 +306,8 @@ def store_curated_record( collection: str, data: BaseModel, class_name: str, - author_id: str | None = None, - api_key: str | None = Depends(api_key_header_scheme), + author_id: str | None, + api_key: str | None, ): instance_state = get_instance_state() with wrap_http_exception( @@ -311,16 +325,17 @@ def store_curated_record( remove_keys=('@type',), ) + sorted_json_object = order_dict(json_object) with wrap_http_exception(CurieResolutionError): backend.add_record( model_store.pid_to_iri(pid), class_name, - json_object, + sorted_json_object, ) for audit_backend in instance_state.audit_backends[collection]: audit_backend.add_record( - record=json_object, + record=sorted_json_object, committer_id=auth_info.user_id, author_id=author_id, ) diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index 0de564e..ed94359 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging from typing import TYPE_CHECKING, Annotated from fastapi import ( @@ -13,6 +12,7 @@ from fastapi_pagination import ( add_pagination, paginate, ) +from starlette.status import HTTP_403_FORBIDDEN from dump_things_service import ( HTTP_401_UNAUTHORIZED, @@ -28,7 +28,11 @@ from dump_things_service.abstract_config import ( from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError -from dump_things_service.instance_state import get_instance_state +from dump_things_service.instance_state import ( + InstanceState, + check_instance_state_collection, + get_instance_state, +) from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, @@ -36,6 +40,7 @@ from dump_things_service.utils import ( cleaned_json, create_token_store, get_on_disk_labels, + order_dict, wrap_http_exception, ) @@ -47,7 +52,6 @@ if TYPE_CHECKING: from dump_things_service.store.model_store import _ModelStore -logger = logging.getLogger('dump_things_service') router = APIRouter() add_pagination(router) @@ -69,7 +73,7 @@ async def incoming_read_labels( on_disk_labels = get_on_disk_labels( instance_state.store_path, get_config(), collection ) - return list(configured_labels.union(on_disk_labels)) + return sorted(configured_labels.union(on_disk_labels)) @router.get( @@ -85,11 +89,7 @@ async def incoming_read_records_of_type( matching: str | None = None, ): instance_state = get_instance_state() - if class_name not in instance_state.collections[collection].active_classes: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No '{class_name}'-class in collection '{collection}'.", - ) + _check_collection_and_class(instance_state, collection, class_name) return await _incoming_read_records( collection=collection, @@ -115,11 +115,7 @@ async def incoming_read_records_of_type_paginated( matching: str | None = None, ) -> Page[dict]: instance_state = get_instance_state() - if class_name not in instance_state.collections[collection].active_classes: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No '{class_name}'-class in collection '{collection}'.", - ) + _check_collection_and_class(instance_state, collection, class_name) record_list = await _incoming_read_records( collection=collection, @@ -128,10 +124,24 @@ async def incoming_read_records_of_type_paginated( pid=None, matching=matching, api_key=api_key, + upper_bound=None, ) return paginate(record_list) +def _check_collection_and_class( + instance_state: InstanceState, + collection: str, + class_name: str, +): + check_instance_state_collection(instance_state, collection) + if class_name not in instance_state.collections[collection].active_classes: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No '{class_name}'-class in collection '{collection}'.", + ) + + @router.get( '/{collection}/incoming/{label}/records/', tags=['Incoming area: read records'], @@ -223,7 +233,7 @@ async def _incoming_read_records( pid: str | None, matching: str | None = None, api_key: str | None = None, - upper_bound: int = 1000, + upper_bound: int | None = 1000, ) -> LazyList | dict | None: model_store, backend = await _get_store_and_backend(collection, label, api_key) @@ -232,6 +242,13 @@ async def _incoming_read_records( if record_info: return record_info.json_object return None + # TODO: check whether HTTP_404_NOT_FOUND should be raised instead of + # returning an empty HTTP_200_OK result. This will influence on how the + # endpoint is used by the clients. + # raise HTTPException( + # status_code=HTTP_404_NOT_FOUND, + # detail=f'no record with pid: {pid}', + # ) if class_name: result_list = backend.get_records_of_classes([class_name], matching) else: @@ -242,9 +259,9 @@ async def _incoming_read_records( len(result_list), upper_bound, collection, - f'/incoming/{label}/records/p/{class_name}' + f'/incoming/{label}/records/{class_name}' if class_name - else f'/incoming/{label}/records/p/', + else f'/incoming/{label}/records/', ) return ModifierList( @@ -256,7 +273,7 @@ async def _incoming_read_records( async def _incoming_delete_record( collection: str, label: str, - pid: str | None, + pid: str, api_key: str | None = None, ) -> bool: model_store, backend = await _get_store_and_backend(collection, label, api_key) @@ -274,10 +291,10 @@ async def _incoming_delete_record( async def _get_store_and_backend( collection: str, label: str, - plain_token: str | None, + api_key: str | None, ) -> tuple[_ModelStore, StorageBackend]: # Authorize api_key - await authorize_zones(collection, plain_token) + await authorize_zones(collection, api_key) # Check that the incoming zone exists instance_state = get_instance_state() @@ -327,7 +344,7 @@ async def authorize_zones( permissions = auth_info.token_permission if permissions.zones_access is False: raise HTTPException( - status_code=HTTP_401_UNAUTHORIZED, + status_code=HTTP_403_FORBIDDEN, detail=f'no access to incoming zones of collection `{collection}`', ) @@ -337,7 +354,7 @@ async def store_incoming_record( label: str, data: BaseModel, class_name: str, - api_key: str | None = Depends(api_key_header_scheme), + api_key: str | None, ): instance_state = get_instance_state() with wrap_http_exception( @@ -359,9 +376,10 @@ async def store_incoming_record( remove_keys=('@type',), ) + sorted_json_object = order_dict(json_object) with wrap_http_exception(CurieResolutionError): backend.add_record( model_store.pid_to_iri(pid), class_name, - json_object, + sorted_json_object, ) diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index f634ace..a77bb64 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -9,9 +9,11 @@ from typing import ( ) import yaml +from fastapi import HTTPException from pydantic import ValidationError from yaml.scanner import ScannerError +from dump_things_service import HTTP_404_NOT_FOUND from dump_things_service.abstract_config import ( MappingMethod, RecordDirConfigFileContent, @@ -155,3 +157,14 @@ def get_mapping_function_by_name(mapping_function_name: str) -> Callable: def get_mapping_function(collection_config: RecordDirConfigFileContent): return mapping_functions[collection_config.idfx] + + +def check_instance_state_collection( + instance_state: InstanceState, + collection: str, +): + if collection not in instance_state.collections: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No such collection: '{collection}'.", + ) diff --git a/dump_things_service/tests/fixtures.py b/dump_things_service/tests/fixtures.py index 39bd595..a9f02dd 100644 --- a/dump_things_service/tests/fixtures.py +++ b/dump_things_service/tests/fixtures.py @@ -395,6 +395,7 @@ def fastapi_client_simple(fastapi_app_simple): response = test_client.post( '/collections', json=collection_config.model_dump( + exclude_none=True, exclude_unset=True, mode='json', by_alias=True, @@ -407,7 +408,11 @@ def fastapi_client_simple(fastapi_app_simple): for token_config in g_default_tokens: response = test_client.post( '/tokens', - json=token_config.model_dump(exclude_unset=True, mode='json'), + json=token_config.model_dump( + exclude_none=True, + exclude_unset=True, + mode='json', + ), headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == 201 diff --git a/dump_things_service/tests/test_canonical.py b/dump_things_service/tests/test_canonical.py new file mode 100644 index 0000000..bf8d717 --- /dev/null +++ b/dump_things_service/tests/test_canonical.py @@ -0,0 +1,56 @@ +from pathlib import Path + +from dump_things_service import HTTP_200_OK +from dump_things_service.abstract_config import ( + MappingMethod, + mapping_functions, +) + + +def test_canonicalization(fastapi_client_simple): + test_client, store_path, _admin_token = fastapi_client_simple + + pid = 'http://example.com/test_canonicalization/1' + record_a = { + 'pid': pid, + 'given_name': 'Alice', + } + + record_b = { + 'given_name': 'Bob', + 'pid': pid, + } + + # Ensure that the two records are represented in the same order on disk. + # (The current implementation should guarantee that because the pydantic + # objects that are instantiated from the JSON input have a fixed order + # of attributes. Once the implementation is changed, this test here might + # catch regressions). + file_content = [] + for record in (record_a, record_b): + response = test_client.post( + '/collection_1/record/Person', + headers={'x-dumpthings-token': 'token-1'}, + json=record, + ) + assert response.status_code == HTTP_200_OK + file_content.append( + _get_record_dir_record( + store_path, + 'incoming/collection_1/in_token_1/Person', + pid, + MappingMethod.digest_md5, + ), + ) + assert file_content[0] == file_content[1].replace('Bob', 'Alice') + + +def _get_record_dir_record( + store_path: Path, + path_part: str | Path, + pid: str, + mapping_method: MappingMethod, +) -> str: + file_name = mapping_functions[mapping_method](pid, 'yaml') + with (store_path / path_part / file_name).open('rt') as f: + return f.read() diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index 2dcb210..2c50572 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -15,9 +15,11 @@ from contextlib import contextmanager from functools import reduce from typing import ( TYPE_CHECKING, + cast, ) import fsspec +import yaml from fastapi import HTTPException from rdflib import Graph @@ -30,6 +32,7 @@ from dump_things_service import ( ) from dump_things_service.abstract_config import ( Configuration, + RecordDirBackendConfig, TokenModes, TokenPermission, check_collection, @@ -89,12 +92,14 @@ def cleaned_json(data: JSON, remove_keys: tuple[str, ...] = ('@type',)) -> JSON: def combine_ttl(documents: list[str]) -> str: graphs = [Graph().parse(data=doc, format='ttl') for doc in documents] + if not graphs: + return '' return reduce(lambda g1, g2: g1 + g2, graphs).serialize(format='ttl') @contextmanager def wrap_http_exception( - exception_class: type[BaseException] = ValueError, + exception_class: type[Exception] = ValueError, status_code: int = HTTP_400_BAD_REQUEST, header: str = '', ): @@ -154,7 +159,12 @@ def get_on_disk_labels( ) -> set[str]: check_collection(abstract_config, collection) - incoming_path = store_path / abstract_config.collections[collection].incoming + if abstract_config.collections[collection].incoming is None: + return set() + + incoming_path = store_path / cast( + 'Path', abstract_config.collections[collection].incoming + ) if not incoming_path or not incoming_path.exists(): return set() @@ -221,31 +231,26 @@ def get_token_store( abstract_config: Configuration, instance_state: InstanceState, collection_name: str, - token_representation: str | None, + token_representation: str, *, is_token_name: bool = False, -) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]: +) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None]: # If a token representation is provided, try to authenticate the token # with the authentication providers that are associated with the collection. if not is_token_name: - if token_representation is None: - msg = 'get_token_store: token_representation is None and is_token_name is False. This is a calling error!' - logger.error(msg) - raise ValueError(msg) - auth_info = authenticate_token( instance_state, collection_name, token_representation, ) - + token_key = token_representation else: auth_info = get_default_token_auth_info( abstract_config=abstract_config, collection_name=collection_name, token_name=token_representation, ) - token_representation = None + token_key = None if not auth_info: detail = f"invalid token for collection '{collection_name}'" @@ -258,12 +263,12 @@ def get_token_store( # If the token has no incoming-read or incoming-write permissions, we do not # need to create a store. if not permissions.incoming_read and not permissions.incoming_write: - instance_state.incoming_stores[collection_name][token_representation] = ( + instance_state.incoming_stores[collection_name][token_key] = ( None, permissions, auth_info.user_id, ) - return instance_state.incoming_stores[collection_name][token_representation] + return instance_state.incoming_stores[collection_name][token_key] # Check whether the collection has an incoming definition incoming = abstract_config.collections[collection_name].incoming @@ -274,9 +279,7 @@ def get_token_store( ) # Check whether a store for this collection and token does already exist. - store_info = instance_state.incoming_stores[collection_name].get( - token_representation - ) + store_info = instance_state.incoming_stores[collection_name].get(token_key) if store_info: return store_info @@ -288,12 +291,12 @@ def get_token_store( store_dir=store_dir, ) - instance_state.incoming_stores[collection_name][token_representation] = ( + instance_state.incoming_stores[collection_name][token_key] = ( token_store, permissions, auth_info.user_id, ) - return instance_state.incoming_stores[collection_name][token_representation] + return instance_state.incoming_stores[collection_name][token_key] def create_store( @@ -353,7 +356,9 @@ def create_token_store( store_dir=store_dir, order_by=instance_state.order_by, schema_uri=schema_uri, - mapping_function=backend_config.mapping_method, + mapping_function=cast( + 'RecordDirBackendConfig', backend_config + ).mapping_method, suffix='yaml', ) elif backend_name == 'sqlite': @@ -449,7 +454,7 @@ def create_sqlite_token_store_backend( def check_bounds( length: int | None, max_length: int, collection: str, alternative_url: str ): - if length > max_length: + if length is not None and length > max_length: raise HTTPException( status_code=HTTP_413_CONTENT_TOO_LARGE, detail=f"Too many records found in collection '{collection}'. " @@ -477,7 +482,14 @@ async def process_token( abstract_config, instance_state, collection, - api_key, + token_representation=api_key, + is_token_name=False, + ) + + if token_permissions is None: + raise HTTPException( + status_code=HTTP_401_UNAUTHORIZED, + detail=f"No read access to curated or incoming data in collection '{collection}'.", ) final_permissions = join_default_token_permissions( @@ -527,3 +539,30 @@ def var_escape( name: str, ) -> str: return name.replace('_', '___').replace('-', '_0_') + + +def json2yaml( + json_object: dict, +) -> str: + return yaml.dump( + data=json_object, + sort_keys=True, + allow_unicode=True, + default_flow_style=False, + ) + + +def order_dict( + d: dict, +) -> dict: + return cast('dict', _order_dict(d)) + + +def _order_dict( + d: dict | list | str | int | None, +) -> dict | list | str | int | None: + if isinstance(d, dict): + return {k: order_dict(d[k]) for k in sorted(d)} + if isinstance(d, list): + return [order_dict(e) for e in d] + return d