orinoco/dump-things-service
orinoco/dump-things-service
Watch
5
0
Fork
You've already forked dump-things-service
 1
Code Issues 38 Pull requests Packages Activity Actions

Packaging/dev-tools/docs update #244

Merged
cmo merged 19 commits from packaging into master 2026-07-01 09:00:58 +00:00
Conversation 0 Commits 19 Files changed 70 +1518 -1327
70 changed files with 1518 additions and 1327 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.forgejo/workflows/codespell.yml
View file

@ -2,7 +2,7 @@
--- ---
name: Codespell name: Codespell
on: workflow_dispatch on: [push, pull_request, workflow_dispatch]
permissions: permissions:
contents: read contents: read
@ -10,13 +10,13 @@ permissions:
jobs: jobs:
codespell: codespell:
name: Check for spelling errors name: Check for spelling errors
runs-on: ubuntu-latest runs-on: debian-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v5 uses: actions/checkout@v5
- name: Codespell - name: Codespell
uses: codespell-project/actions-codespell@v2 uses: https://github.com/codespell-project/actions-codespell@v2
with: with:
ignore_words_list: crate ignore_words_list: crate

36
.forgejo/workflows/mypy-pr.yml Normal file
View file

@ -0,0 +1,36 @@
name: Type annotation (PR)
on:
pull_request:
paths:
- 'dump_things_service/**.py'
- '!**/tests/**.py'
jobs:
check:
runs-on: debian-latest
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v6
- name: Install hatch
run: uv tool install hatch
- name: Get Python changed files
id: changed-py-files
uses: https://github.com/tj-actions/changed-files@v46
with:
files: |
*.py
**/*.py
- name: Type check changed files
if: steps.changed-py-files.outputs.any_changed == 'true'
run: |
# get any type stubs that mypy thinks it needs
# run mypy on the modified files only, and do not even follow imports.
# this results is a fairly superficial test, but given the overall
# state of annotations, we strive to become more correct incrementally
# with focused error reports, rather than barfing a huge complaint
# that is unrelated to the changeset someone has been working on.
# run on the oldest supported Python version
hatch run types:mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context ${{ steps.changed-py-files.outputs.all_changed_files }}

17
.forgejo/workflows/ruff.yml Normal file
View file

@ -0,0 +1,17 @@
name: Ruff
on: [push, pull_request]
jobs:
ruff:
name: Code linting
runs-on: debian-latest
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v6
- name: Install hatch
run: uv tool install hatch
- name: Check code
run: hatch check code
- name: Check formatting
run: hatch check fmt

9
.forgejo/workflows/run_tests.yaml
View file

@ -12,9 +12,6 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
- name: Install uv - name: Install uv
uses: astral-sh/setup-uv@v6 uses: astral-sh/setup-uv@v6
@ -23,14 +20,14 @@ jobs:
- name: Run tests - name: Run tests
run: | run: |
hatch run tests:run \ hatch test \
--ignore=dump_things_service/tests/test_generators.py \ --ignore=dump_things_service/tests/test_generators.py \
--ignore=dump_things_service/tests/test_ifabsent_patch.py --ignore=dump_things_service/tests/test_ifabsent_patch.py
- name: Run generator tests - name: Run generator tests
run: | run: |
hatch run tests:run dump_things_service/tests/test_generators.py hatch test dump_things_service/tests/test_generators.py
- name: Run ifabsent-patch tests - name: Run ifabsent-patch tests
run: | run: |
hatch run tests:run dump_things_service/tests/test_ifabsent_patch.py hatch test dump_things_service/tests/test_ifabsent_patch.py

2
.gitignore vendored
View file

@ -3,3 +3,5 @@ dist/**
tmp/** tmp/**
**/__pycache__ **/__pycache__
**/.hypothesis **/.hypothesis
.*.swp
dump_things_service/_version.py

31
.readthedocs.yaml Normal file
View file

@ -0,0 +1,31 @@
# .readthedocs.yaml
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
# Set the OS, Python version and other tools you might need
build:
os: ubuntu-24.04
tools:
python: "3.12"
# Build documentation in the "docs/" directory with Sphinx
sphinx:
configuration: docs/conf.py
# Optionally build your docs in additional formats such as PDF and ePub
# formats:
# - pdf
# - epub
# Optional but recommended, declare the Python requirements required
# to build your documentation
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
python:
install:
- method: pip
path: .
extra_requirements:
- docs

2
CHANGELOG.md
View file

@ -123,7 +123,7 @@
3. The top-level mapping `admin_tokens` was added. 3. The top-level mapping `admin_tokens` was added.
- Configuration files are no longer read when the service is started. Instead - Configuration files are no longer read when the service is started. Instead
the service reads its configuration from the store, if it is present. Thw tool the service reads its configuration from the store, if it is present. The tool
(`dump-things-load-config`) can read an existing configuration (`dump-things-load-config`) can read an existing configuration
file and manifest the described configuration on a running dump-things server. file and manifest the described configuration on a running dump-things server.
It supports pre version 6 config files and converts them to the new It supports pre version 6 config files and converts them to the new

2
README.md
View file

@ -594,7 +594,7 @@ Most endpoints require a *collection*. These correspond to the names of the "dat
The service provides the following user endpoints (In addition to user endpoints, there exist endpoints for curators. To view them, check the `/docs`-path in an installed service): The service provides the following user endpoints (In addition to user endpoints, there exist endpoints for curators. To view them, check the `/docs`-path in an installed service):
- `POST /maintenance`: this endpoint allows to set a collection into mantenance mode. - `POST /maintenance`: this endpoint allows to set a collection into maintenance mode.
In maintenance mode, only tokens with curator-privileges can access the collection. In maintenance mode, only tokens with curator-privileges can access the collection.
The posted data is a JSON that contains the name of the collection and whether the maintenance state should be active or not, for example: The posted data is a JSON that contains the name of the collection and whether the maintenance state should be active or not, for example:
```json ```json

2
docs/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
_build
generated

20
docs/Makefile Normal file
View file

@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?= --fail-on-warning
SPHINXBUILD ?= sphinx-build
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

0
docs/_static/.gitkeep vendored Normal file
View file

48
docs/conf.py Normal file
View file

@ -0,0 +1,48 @@
# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
import dump_things_service
project = 'dump-things-server'
copyright = '2025-2026, Christian Mönch'
author = 'Christian Mönch'
release = dump_things_service.__version__
# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [
'sphinx.ext.autosummary',
'sphinx.ext.autodoc',
'sphinx_autodoc_typehints',
'sphinx.ext.viewcode',
]
templates_path = ['_templates']
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
primary_domain = 'py'
autoclass_content = "both"
typehints_use_signature = True
typehints_use_signature_return = True
# we build some docstrings from loguru. define some no-op substitutions
# to avoid errors
rst_prolog = """
.. |Logger| replace:: Logger
.. |add| replace:: add
.. |sys.stderr| replace:: sys.stderr
.. |str.format| replace:: str.format
"""
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
html_theme = 'sphinx_rtd_theme'
html_static_path = ['_static']

10
docs/index.rst Normal file
View file

@ -0,0 +1,10 @@
The `dump-thing-server` documentation
=====================================
HERE BE CONTENT...
Indices and tables
==================
* :ref:`genindex`
* :ref:`search`

5
dump_things_service/__init__.py
View file

@ -20,8 +20,9 @@ from starlette.status import (
HTTP_503_SERVICE_UNAVAILABLE, HTTP_503_SERVICE_UNAVAILABLE,
) )
from dump_things_service._version import __version__
__all__ = [ __all__ = [
'Format',
'HTTP_200_OK', 'HTTP_200_OK',
'HTTP_201_CREATED', 'HTTP_201_CREATED',
'HTTP_300_MULTIPLE_CHOICES', 'HTTP_300_MULTIPLE_CHOICES',
@ -37,6 +38,8 @@ __all__ = [
'HTTP_503_SERVICE_UNAVAILABLE', 'HTTP_503_SERVICE_UNAVAILABLE',
'JSON', 'JSON',
'YAML', 'YAML',
'Format',
'__version__',
'config_file_name', 'config_file_name',
'reserved_collection_names', 'reserved_collection_names',
] ]

108
dump_things_service/abstract_config.py
View file

@ -1,14 +1,13 @@
import enum import enum
import hashlib import hashlib
import logging import logging
from collections.abc import Callable, Iterable
from functools import partial from functools import partial
from pathlib import ( from pathlib import (
Path, Path,
PurePosixPath, PurePosixPath,
) )
from typing import ( from typing import (
Callable,
Iterable,
Literal, Literal,
cast, cast,
) )
@ -17,7 +16,8 @@ from fastapi import HTTPException
from pydantic import ( from pydantic import (
BaseModel, BaseModel,
ConfigDict, ConfigDict,
Field, ValidationError, Field,
ValidationError,
) )
from yaml.scanner import ScannerError from yaml.scanner import ScannerError
@ -27,12 +27,11 @@ from dump_things_service import (
) )
from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.audit.gitaudit import GitAuditBackend
from dump_things_service.backends.record_dir import ( from dump_things_service.backends.record_dir import (
_RecordDirStore,
RecordDirStore, RecordDirStore,
_RecordDirStore,
) )
from dump_things_service.exceptions import ConfigError from dump_things_service.exceptions import ConfigError
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
g_abstract_configuration = None g_abstract_configuration = None
@ -103,7 +102,9 @@ class CollectionConfig(BaseModel):
curated: PurePosixPath curated: PurePosixPath
schema_location: str = Field(alias='schema') schema_location: str = Field(alias='schema')
incoming: PurePosixPath | None = None incoming: PurePosixPath | None = None
backend: RecordDirBackendConfig | SQLiteBackendConfig = RecordDirBackendConfig(type='record_dir+stl') backend: RecordDirBackendConfig | SQLiteBackendConfig = RecordDirBackendConfig(
type='record_dir+stl'
)
auth_sources: list[ForgejoAuthSpec | ConfigAuthSpec] = [ConfigAuthSpec()] auth_sources: list[ForgejoAuthSpec | ConfigAuthSpec] = [ConfigAuthSpec()]
audit_backends: list[GitAuditBackendConfig] = [] audit_backends: list[GitAuditBackendConfig] = []
submission_tags: TagSpec = TagSpec() submission_tags: TagSpec = TagSpec()
@ -200,7 +201,7 @@ def get_token_permissions(mode: str) -> TokenPermission:
def get_config_backends( def get_config_backends(
store_path: Path, store_path: Path,
) -> tuple[_RecordDirStore, GitAuditBackend]: ) -> tuple[_RecordDirStore, GitAuditBackend]:
global config_audit global config_audit
global config_backend global config_backend
@ -211,9 +212,7 @@ def get_config_backends(
if config_backend is None: if config_backend is None:
config_backend = RecordDirStore( config_backend = RecordDirStore(
config_path, config_path, mapping_functions[MappingMethod.digest_md5], 'yaml'
mapping_functions[MappingMethod.digest_md5],
'yaml'
) )
audit_path = store_path / config_audit_path audit_path = store_path / config_audit_path
@ -226,8 +225,8 @@ def get_config_backends(
def read_config( def read_config(
store_path: Path, store_path: Path,
force_reload: bool = False, force_reload: bool = False,
) -> Configuration: ) -> Configuration:
global g_abstract_configuration global g_abstract_configuration
@ -244,7 +243,7 @@ def read_config(
if record_info if record_info
else Configuration( else Configuration(
type='collections', type='collections',
version = 2, version=2,
) )
) )
except ValidationError as ve: except ValidationError as ve:
@ -259,12 +258,12 @@ def get_config() -> Configuration:
if not g_abstract_configuration: if not g_abstract_configuration:
msg = 'Configuration not yet loaded' msg = 'Configuration not yet loaded'
raise RuntimeError(msg) raise RuntimeError(msg)
return cast(Configuration, g_abstract_configuration) return cast('Configuration', g_abstract_configuration)
def store_config( def store_config(
store_path, store_path,
config: Configuration, config: Configuration,
): ):
global g_abstract_configuration global g_abstract_configuration
@ -274,7 +273,7 @@ def store_config(
config_backend.add_record( config_backend.add_record(
iri=dump_things_config_iri, iri=dump_things_config_iri,
class_name='DumpThingsConfig', class_name='DumpThingsConfig',
json_object=json_object json_object=json_object,
) )
audit_backend.add_record( audit_backend.add_record(
record=json_object, record=json_object,
@ -284,8 +283,8 @@ def store_config(
def tokens_for_collection( def tokens_for_collection(
config: Configuration, config: Configuration,
collection: str, collection: str,
) -> Iterable[TokenConfig]: ) -> Iterable[TokenConfig]:
yield from ( yield from (
token token
@ -295,8 +294,8 @@ def tokens_for_collection(
def check_collection( def check_collection(
abstract_config: Configuration, abstract_config: Configuration,
collection: str, collection: str,
): ):
if collection not in abstract_config.collections: if collection not in abstract_config.collections:
raise HTTPException( raise HTTPException(
@ -306,18 +305,17 @@ def check_collection(
def check_label( def check_label(
store_path: Path, store_path: Path,
abstract_config: Configuration, abstract_config: Configuration,
collection: str, collection: str,
label: str, label: str,
): ):
from dump_things_service.utils import get_on_disk_labels from dump_things_service.utils import get_on_disk_labels
"""Check that a label exists in a collection configuration or on disk""" """Check that a label exists in a collection configuration or on disk"""
if ( if label not in get_config_labels(
label not in get_config_labels(abstract_config, collection) abstract_config, collection
and label not in get_on_disk_labels(store_path, abstract_config, collection) ) and label not in get_on_disk_labels(store_path, abstract_config, collection):
):
raise HTTPException( raise HTTPException(
status_code=HTTP_404_NOT_FOUND, status_code=HTTP_404_NOT_FOUND,
detail=f"No incoming label: '{label}' in collection: '{collection}'.", detail=f"No incoming label: '{label}' in collection: '{collection}'.",
@ -325,8 +323,8 @@ def check_label(
def get_config_labels( def get_config_labels(
abstract_config: Configuration, abstract_config: Configuration,
collection: str, collection: str,
) -> set[str]: ) -> set[str]:
check_collection(abstract_config, collection) check_collection(abstract_config, collection)
return { return {
@ -336,17 +334,14 @@ def get_config_labels(
} }
def get_default_token_name( def get_default_token_name(abstract_config: Configuration, collection: str) -> str:
abstract_config: Configuration,
collection: str
) -> str:
check_collection(abstract_config, collection) check_collection(abstract_config, collection)
return abstract_config.collections[collection].default_token return abstract_config.collections[collection].default_token
def get_token_info_by_representation( def get_token_info_by_representation(
abstract_config: Configuration, abstract_config: Configuration,
token_representation: str, token_representation: str,
) -> tuple[str, TokenConfig] | None: ) -> tuple[str, TokenConfig] | None:
"""Get the name of the token given in `token_representation`""" """Get the name of the token given in `token_representation`"""
hashed_representation = hash_token_representation(token_representation) hashed_representation = hash_token_representation(token_representation)
@ -361,23 +356,22 @@ def get_token_info_by_representation(
def hash_token_representation( def hash_token_representation(
token_representation: str, token_representation: str,
) -> str: ) -> str:
return hashlib.sha256(token_representation.encode()).hexdigest() return hashlib.sha256(token_representation.encode()).hexdigest()
def get_token_config_by_name( def get_token_config_by_name(
abstract_config: Configuration, abstract_config: Configuration,
token_name: str, token_name: str,
) -> TokenConfig | None: ) -> TokenConfig | None:
return abstract_config.tokens.get(token_name) return abstract_config.tokens.get(token_name)
def get_token_infos_for_collection( def get_token_infos_for_collection(
abstract_config: Configuration, abstract_config: Configuration,
collection_name: str, collection_name: str,
) -> Iterable[tuple[str, TokenConfig, TokenCollectionConfig]]: ) -> Iterable[tuple[str, TokenConfig, TokenCollectionConfig]]:
yield from { yield from {
(token_name, token_config, token_collection_config) (token_name, token_config, token_collection_config)
for token_name, token_config in abstract_config.tokens.items() for token_name, token_config in abstract_config.tokens.items()
@ -387,11 +381,10 @@ def get_token_infos_for_collection(
def get_token_config_for_representation_and_collection( def get_token_config_for_representation_and_collection(
abstract_config: Configuration, abstract_config: Configuration,
collection_name: str, collection_name: str,
token_representation: str, token_representation: str,
) -> tuple[str, TokenConfig, TokenCollectionConfig] | None: ) -> tuple[str, TokenConfig, TokenCollectionConfig] | None:
token_info = get_token_info_by_representation( token_info = get_token_info_by_representation(
abstract_config=abstract_config, abstract_config=abstract_config,
token_representation=token_representation, token_representation=token_representation,
@ -405,8 +398,8 @@ def get_token_config_for_representation_and_collection(
def get_collection_config_by_name( def get_collection_config_by_name(
abstract_config: Configuration, abstract_config: Configuration,
collection_name: str, collection_name: str,
) -> CollectionConfig: ) -> CollectionConfig:
collection_config = abstract_config.collections.get(collection_name) collection_config = abstract_config.collections.get(collection_name)
if not collection_config: if not collection_config:
@ -418,10 +411,9 @@ def get_collection_config_by_name(
def get_default_token_config( def get_default_token_config(
abstract_config: Configuration, abstract_config: Configuration,
collection: str, collection: str,
) -> TokenConfig | None: ) -> TokenConfig | None:
default_token_name = get_collection_config_by_name( default_token_name = get_collection_config_by_name(
abstract_config, abstract_config,
collection, collection,
@ -445,18 +437,18 @@ def get_hex_digest(hasher: Callable, data: str) -> str:
def mapping_digest_p3( def mapping_digest_p3(
hasher: Callable, hasher: Callable,
pid: str, pid: str,
suffix: str, suffix: str,
) -> Path: ) -> Path:
hex_digest = get_hex_digest(hasher, pid) hex_digest = get_hex_digest(hasher, pid)
return Path(hex_digest[:3]) / (hex_digest[3:] + '.' + suffix) return Path(hex_digest[:3]) / (hex_digest[3:] + '.' + suffix)
def mapping_digest_p3_p3( def mapping_digest_p3_p3(
hasher: Callable, hasher: Callable,
pid: str, pid: str,
suffix: str, suffix: str,
) -> Path: ) -> Path:
hex_digest = get_hex_digest(hasher, pid) hex_digest = get_hex_digest(hasher, pid)
return Path(hex_digest[:3]) / hex_digest[3:6] / (hex_digest[6:] + '.' + suffix) return Path(hex_digest[:3]) / hex_digest[3:6] / (hex_digest[6:] + '.' + suffix)

7
dump_things_service/admin.py
View file

@ -9,14 +9,13 @@ from dump_things_service.abstract_config import (
) )
from dump_things_service.instance_state import InstanceState from dump_things_service.instance_state import InstanceState
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
def authenticate_admin( def authenticate_admin(
instance_state: InstanceState, instance_state: InstanceState,
abstract_config: Configuration, abstract_config: Configuration,
api_key: str, api_key: str,
): ):
if api_key: if api_key:
hashed_token_representation = hash_token_representation(api_key) hashed_token_representation = hash_token_representation(api_key)

12
dump_things_service/audit/__init__.py
View file

@ -7,10 +7,10 @@ from abc import (
class AuditBackend(metaclass=ABCMeta): class AuditBackend(metaclass=ABCMeta):
@abstractmethod @abstractmethod
def add_record( def add_record(
self, self,
record: dict, record: dict,
committer_id: str, committer_id: str,
author_id: str | None = None, author_id: str | None = None,
) -> None: ) -> None:
"""Add information about a new record version to the audit log """Add information about a new record version to the audit log
@ -35,8 +35,8 @@ class AuditBackend(metaclass=ABCMeta):
@abstractmethod @abstractmethod
def get_audit_log( def get_audit_log(
self, self,
record_id: str, record_id: str,
) -> dict: ) -> dict:
"""Get the content of the audit log """Get the content of the audit log

162
dump_things_service/audit/gitaudit.py
View file

@ -6,6 +6,7 @@ committed.
Changes are annotated with a time stamp and a user-id Changes are annotated with a time stamp and a user-id
""" """
from __future__ import annotations from __future__ import annotations
import hashlib import hashlib
@ -23,21 +24,20 @@ import yaml
from datalad_core.git_utils import apply_changeset from datalad_core.git_utils import apply_changeset
from datalad_core.repo import Repo from datalad_core.repo import Repo
from datalad_core.runners import ( from datalad_core.runners import (
call_git,
CommandError, CommandError,
call_git,
) )
from . import AuditBackend from dump_things_service.audit import AuditBackend
index_file_name = 'gitaudit_index.log' index_file_name = 'gitaudit_index.log'
class FlushingThread(Thread): class FlushingThread(Thread):
def __init__( def __init__(
self, self,
backend: GitAuditBackend, backend: GitAuditBackend,
auto_flush_timeout: int, auto_flush_timeout: int,
): ):
super().__init__() super().__init__()
self.auto_flush_timeout = auto_flush_timeout self.auto_flush_timeout = auto_flush_timeout
@ -56,11 +56,10 @@ class FlushingThread(Thread):
class GitAuditBackend(AuditBackend): class GitAuditBackend(AuditBackend):
def __init__( def __init__(
self, self,
path: Path, path: Path,
auto_flush_timeout: int = 60, auto_flush_timeout: int = 60,
): ):
self.path = path self.path = path
self.index_path = None self.index_path = None
@ -69,7 +68,8 @@ class GitAuditBackend(AuditBackend):
self.lock = Lock() self.lock = Lock()
self.last_flush_time = 0 self.last_flush_time = 0
if auto_flush_timeout < 1: if auto_flush_timeout < 1:
raise ValueError('auto_flush_timeout must be greater or equal to 1') msg = 'auto_flush_timeout must be greater or equal to 1'
raise ValueError(msg)
self.flushing_thread = FlushingThread(self, auto_flush_timeout) self.flushing_thread = FlushingThread(self, auto_flush_timeout)
self.flushing_thread.start() self.flushing_thread.start()
self._init_repo() self._init_repo()
@ -82,10 +82,10 @@ class GitAuditBackend(AuditBackend):
self.flushing_thread = None self.flushing_thread = None
def add_record( def add_record(
self, self,
record: dict, record: dict,
committer_id: str, committer_id: str,
author_id: str | None = None, author_id: str | None = None,
) -> None: ) -> None:
with self.lock: with self.lock:
author_id = committer_id if author_id is None else author_id author_id = committer_id if author_id is None else author_id
@ -109,15 +109,15 @@ class GitAuditBackend(AuditBackend):
self.last_flush_time = time.time() self.last_flush_time = time.time()
def get_audit_log( def get_audit_log(
self, self,
record_id: str, record_id: str,
) -> dict: ) -> dict:
with self.lock: with self.lock:
return self._locked_get_audit_log(record_id) return self._locked_get_audit_log(record_id)
def _locked_get_audit_log( def _locked_get_audit_log(
self, self,
record_id: str, record_id: str,
) -> dict: ) -> dict:
self._locked_flush() self._locked_flush()
@ -125,32 +125,42 @@ class GitAuditBackend(AuditBackend):
# the records # the records
changes = [] changes = []
yaml_location, log_location = map(str, self._get_location_for(record_id)[1:]) yaml_location, log_location = map(str, self._get_location_for(record_id)[1:])
commit_hashes = call_git( commit_hashes = (
['log', '--format=%H', '--', log_location], call_git(
cwd=self.path, ['log', '--format=%H', '--', log_location],
capture_output=True,
).decode().splitlines()
for commit_hash in commit_hashes:
log_diff_lines = call_git(
['show', '--format=%b', commit_hash, '--', log_location],
cwd=self.path, cwd=self.path,
capture_output=True, capture_output=True,
).decode().splitlines() )
.decode()
.splitlines()
)
for commit_hash in commit_hashes:
log_diff_lines = (
call_git(
['show', '--format=%b', commit_hash, '--', log_location],
cwd=self.path,
capture_output=True,
)
.decode()
.splitlines()
)
# Get the log entry # Get the log entry
log_line = tuple( log_line = next(filter(
filter(
lambda l: not l.startswith('+++') and l.startswith('+'), lambda l: not l.startswith('+++') and l.startswith('+'),
log_diff_lines, log_diff_lines,
) ))[1:]
)[0][1:]
log_entry = json.loads(log_line) log_entry = json.loads(log_line)
# Get the YAML diff # Get the YAML diff
yaml_diff_lines = call_git( yaml_diff_lines = (
['show', '--format=%b', commit_hash, '--', yaml_location], call_git(
cwd=self.path, ['show', '--format=%b', commit_hash, '--', yaml_location],
capture_output=True, cwd=self.path,
).decode().splitlines() capture_output=True,
)
.decode()
.splitlines()
)
yaml_diff = '\n'.join(filter(lambda l: l != '', yaml_diff_lines)) + '\n' yaml_diff = '\n'.join(filter(lambda l: l != '', yaml_diff_lines)) + '\n'
# Get the YAML content # Get the YAML content
@ -173,15 +183,15 @@ class GitAuditBackend(AuditBackend):
return {c[0]: c[1:] for c in changes} return {c[0]: c[1:] for c in changes}
def get_audit_logs( def get_audit_logs(
self, self,
record_id_pattern: str, record_id_pattern: str,
) -> dict: ) -> dict:
with self.lock: with self.lock:
return self._locked_get_audit_logs(record_id_pattern) return self._locked_get_audit_logs(record_id_pattern)
def _locked_get_audit_logs( def _locked_get_audit_logs(
self, self,
record_id_pattern: str, record_id_pattern: str,
) -> dict: ) -> dict:
self._locked_flush() self._locked_flush()
matcher = re.compile(record_id_pattern) matcher = re.compile(record_id_pattern)
@ -197,12 +207,12 @@ class GitAuditBackend(AuditBackend):
} }
def _add_elements( def _add_elements(
self, self,
record_id: str, record_id: str,
location: tuple[str, Path, Path], location: tuple[str, Path, Path],
committer_id: str, committer_id: str,
author_id: str, author_id: str,
record: dict, record: dict,
) -> bool: ) -> bool:
existing_record = self._read_record_from_repo_path(location[1]) existing_record = self._read_record_from_repo_path(location[1])
if existing_record != record: if existing_record != record:
@ -218,10 +228,10 @@ class GitAuditBackend(AuditBackend):
return False return False
def _add_log_entry( def _add_log_entry(
self, self,
log_location: Path, log_location: Path,
committer_id: str, committer_id: str,
author_id: str, author_id: str,
) -> None: ) -> None:
time_stamp = datetime.now().isoformat() time_stamp = datetime.now().isoformat()
entry = { entry = {
@ -234,20 +244,20 @@ class GitAuditBackend(AuditBackend):
self.current_change_set[log_location] = log_content self.current_change_set[log_location] = log_content
def _add_index_entry( def _add_index_entry(
self, self,
record_id: str, record_id: str,
): ):
if record_id not in self.index: if record_id not in self.index:
self.cached_index_entries.append(record_id) self.cached_index_entries.append(record_id)
self.index.add(record_id) self.index.add(record_id)
def _read_from_repo_path( def _read_from_repo_path(
self, self,
path: Path, path: Path,
) -> bytes: ) -> bytes:
try: try:
return call_git( return call_git(
['cat-file', '-p', f'master:{str(path)}'], ['cat-file', '-p', f'master:{path!s}'],
cwd=self.path, cwd=self.path,
capture_output=True, capture_output=True,
) )
@ -257,14 +267,14 @@ class GitAuditBackend(AuditBackend):
raise raise
def _read_record_from_repo_path( def _read_record_from_repo_path(
self, self,
path: Path, path: Path,
): ):
return yaml.safe_load(self._read_from_repo_path(path)) return yaml.safe_load(self._read_from_repo_path(path))
def _has_pending_changes( def _has_pending_changes(
self, self,
location: tuple[str, Path, Path], location: tuple[str, Path, Path],
) -> bool: ) -> bool:
log_pending = location[1] in self.current_change_set log_pending = location[1] in self.current_change_set
record_pending = location[2] in self.current_change_set record_pending = location[2] in self.current_change_set
@ -286,11 +296,11 @@ class GitAuditBackend(AuditBackend):
self.current_change_set = {} self.current_change_set = {}
def _get_location_for( def _get_location_for(
self, self,
record_id: str, record_id: str,
) -> tuple[str, Path, Path]: ) -> tuple[str, Path, Path]:
base = hashlib.sha1(record_id.encode()).hexdigest() base = hashlib.sha1(record_id.encode()).hexdigest()
dir_1, dir_2, name = base[0:3], base[3:6], base[6:] dir_1, dir_2, _name = base[0:3], base[3:6], base[6:]
location_dir = Path(dir_1) / Path(dir_2) location_dir = Path(dir_1) / Path(dir_2)
return ( return (
base, base,
@ -321,28 +331,32 @@ class GitAuditBackend(AuditBackend):
if not self.index_path.exists(): if not self.index_path.exists():
self._rebuild_index() self._rebuild_index()
with open(self.index_path, 'rt') as f: with open(self.index_path) as f:
self.index = set(line.strip() for line in f.readlines()) self.index = {line.strip() for line in f}
def _add_to_index( def _add_to_index(
self, self,
record_id: str, record_id: str,
): ):
if record_id not in self.index: if record_id not in self.index:
self.cached_index_entries.append(record_id) self.cached_index_entries.append(record_id)
self.index.add(record_id) self.index.add(record_id)
def _rebuild_index(self): def _rebuild_index(self):
tree_entries = call_git( tree_entries = (
['ls-tree', '-r', 'master:'], call_git(
cwd=self.path, ['ls-tree', '-r', 'master:'],
capture_output=True, cwd=self.path,
).decode().splitlines() capture_output=True,
with open(self.index_path, 'wt') as f: )
.decode()
.splitlines()
)
with open(self.index_path, 'w') as f:
for line in tree_entries: for line in tree_entries:
if not line.endswith('.yaml'): if not line.endswith('.yaml'):
continue continue
flag, object_type, object_hash, file_name = line.split(maxsplit=3) _flag, _object_type, object_hash, _file_name = line.split(maxsplit=3)
record = yaml.safe_load( record = yaml.safe_load(
call_git( call_git(
['show', object_hash], ['show', object_hash],

17
dump_things_service/audit/tests/test_gitaudit.py
View file

@ -19,7 +19,7 @@ def _get_audit_log_lines(backend: GitAuditBackend, record_id: str) -> list[str]:
def test_gitaudit_basic(tmp_path_factory): def test_gitaudit_basic(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp("gitaudit_backend") tmp_path = tmp_path_factory.mktemp('gitaudit_backend')
backend = GitAuditBackend(tmp_path) backend = GitAuditBackend(tmp_path)
@ -44,14 +44,13 @@ def test_gitaudit_basic(tmp_path_factory):
# Check that the changes are reported # Check that the changes are reported
changes = backend.get_audit_log(record_id) changes = backend.get_audit_log(record_id)
assert len(changes) == 4 assert len(changes) == 4
assert tuple(map(lambda e: e[0:2], changes.values())) == tuple( assert tuple(e[0:2] for e in changes.values()) == tuple(
(f'committer_{100 + i}@x.org', f'author_{i}@y.org') (f'committer_{100 + i}@x.org', f'author_{i}@y.org') for i in range(4)
for i in range(4)
) )
def test_gitaudit_identical_change(tmp_path_factory): def test_gitaudit_identical_change(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp("gitaudit_backend") tmp_path = tmp_path_factory.mktemp('gitaudit_backend')
backend = GitAuditBackend(tmp_path) backend = GitAuditBackend(tmp_path)
@ -59,13 +58,13 @@ def test_gitaudit_identical_change(tmp_path_factory):
backend.add_record( backend.add_record(
record={'pid': record_id}, record={'pid': record_id},
committer_id='committer_b@x.org', committer_id='committer_b@x.org',
author_id = 'author_b@y.org', author_id='author_b@y.org',
) )
backend.add_record( backend.add_record(
record={'pid': record_id}, record={'pid': record_id},
committer_id='committer_b@x.org', committer_id='committer_b@x.org',
author_id = 'author_b@y.org', author_id='author_b@y.org',
) )
# Check that there is only one entry in the audit log # Check that there is only one entry in the audit log
@ -83,7 +82,7 @@ def test_gitaudit_identical_change(tmp_path_factory):
def test_gitaudit_huge_log(tmp_path_factory): def test_gitaudit_huge_log(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp("gitaudit_backend") tmp_path = tmp_path_factory.mktemp('gitaudit_backend')
backend = GitAuditBackend(tmp_path) backend = GitAuditBackend(tmp_path)
@ -96,7 +95,7 @@ def test_gitaudit_huge_log(tmp_path_factory):
backend.add_record( backend.add_record(
record={'pid': record_id, 'content': f'j:{j}, i:{i}'}, record={'pid': record_id, 'content': f'j:{j}, i:{i}'},
committer_id='committer@x.org', committer_id='committer@x.org',
author_id = 'author@y.org', author_id='author@y.org',
) )
# Check that the changes are reported # Check that the changes are reported

1
dump_things_service/auth/__init__.py
View file

@ -8,6 +8,7 @@ determine:
- the incoming_label to be used with the token - the incoming_label to be used with the token
""" """
from __future__ import annotations from __future__ import annotations
import abc import abc

23
dump_things_service/auth/config.py
View file

@ -1,31 +1,30 @@
"""Use configuration information to fetch token permissions, ids, and incoming_label """ """Use configuration information to fetch token permissions, ids, and incoming_label"""
from dump_things_service.abstract_config import Configuration from dump_things_service.abstract_config import (
Configuration,
get_token_config_for_representation_and_collection,
get_token_permissions,
)
from dump_things_service.auth import ( from dump_things_service.auth import (
AuthenticationInfo, AuthenticationInfo,
AuthenticationSource, AuthenticationSource,
InvalidTokenError, InvalidTokenError,
) )
from dump_things_service.abstract_config import (
get_token_permissions,
get_token_config_for_representation_and_collection,
)
class ConfigAuthenticationSource(AuthenticationSource): class ConfigAuthenticationSource(AuthenticationSource):
def __init__( def __init__(
self, self,
abstract_configuration: Configuration, abstract_configuration: Configuration,
collection_name: str, collection_name: str,
): ):
self.abstract_configuration = abstract_configuration self.abstract_configuration = abstract_configuration
self.collection_name = collection_name self.collection_name = collection_name
def authenticate( def authenticate(
self, self,
token_representation: str, token_representation: str,
) -> AuthenticationInfo: ) -> AuthenticationInfo:
result = get_token_config_for_representation_and_collection( result = get_token_config_for_representation_and_collection(
self.abstract_configuration, self.abstract_configuration,
self.collection_name, self.collection_name,

49
dump_things_service/auth/forgejo.py
View file

@ -7,13 +7,14 @@ Note: for some reason, the request:
does not require a token. If the owner and the repo are known, the request does not require a token. If the owner and the repo are known, the request
will emit a complete repository-record including the complete owner-record. will emit a complete repository-record including the complete owner-record.
""" """
from __future__ import annotations from __future__ import annotations
import hashlib import hashlib
import logging import logging
import time import time
from functools import wraps from functools import wraps
from typing import Callable from typing import TYPE_CHECKING
import requests import requests
from requests.exceptions import Timeout from requests.exceptions import Timeout
@ -22,13 +23,16 @@ from dump_things_service import (
HTTP_300_MULTIPLE_CHOICES, HTTP_300_MULTIPLE_CHOICES,
HTTP_401_UNAUTHORIZED, HTTP_401_UNAUTHORIZED,
) )
from dump_things_service.abstract_config import TokenPermission
from dump_things_service.auth import ( from dump_things_service.auth import (
AuthenticationError, AuthenticationError,
AuthenticationInfo, AuthenticationInfo,
AuthenticationSource, AuthenticationSource,
InvalidTokenError, InvalidTokenError,
) )
from dump_things_service.abstract_config import TokenPermission
if TYPE_CHECKING:
from collections.abc import Callable
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
@ -46,7 +50,8 @@ class MethodCache:
def cache_temporary( def cache_temporary(
duration: int = 300, duration: int = 300,
) -> Callable: ) -> Callable:
""" Cache results for a given time (default: 300 seconds) """ """Cache results for a given time (default: 300 seconds)"""
def decorator(func: Callable) -> Callable: def decorator(func: Callable) -> Callable:
@wraps(func) @wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
@ -56,12 +61,15 @@ class MethodCache:
if cached_data is None or time.time() - cached_data[0] > duration: if cached_data is None or time.time() - cached_data[0] > duration:
self.__cached_data[key] = (time.time(), func(*args, **kwargs)) self.__cached_data[key] = (time.time(), func(*args, **kwargs))
return self.__cached_data[key][1] return self.__cached_data[key][1]
return wrapper return wrapper
return decorator return decorator
class RemoteAuthenticationError(AuthenticationError): class RemoteAuthenticationError(AuthenticationError):
"""Exception for remote authentication errors.""" """Exception for remote authentication errors."""
def __init__(self, status: int, message: str): def __init__(self, status: int, message: str):
self.status = status self.status = status
self.message = message self.message = message
@ -133,14 +141,15 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
) from e ) from e
if r.status_code >= HTTP_300_MULTIPLE_CHOICES: if r.status_code >= HTTP_300_MULTIPLE_CHOICES:
msg = f'invalid token: ({r.status_code}): {r.text}' cleaned_text = r.text.replace(token, '***')
msg = f'invalid token: ({r.status_code}): {cleaned_text}'
raise InvalidTokenError(msg) raise InvalidTokenError(msg)
return r.json() return r.json()
@MethodCache.cache_temporary(duration=120) @MethodCache.cache_temporary(duration=120)
def _get_user( def _get_user(
self, self,
token: str, token: str,
) -> dict: ) -> dict:
return self._get_json_from_endpoint('user', token) return self._get_json_from_endpoint('user', token)
@ -183,8 +192,8 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
@staticmethod @staticmethod
def _get_permissions( def _get_permissions(
code_permission: str, code_permission: str,
action_permission: str, action_permission: str,
) -> TokenPermission: ) -> TokenPermission:
is_curator = action_permission == 'write' is_curator = action_permission == 'write'
read = code_permission in ('read', 'write') or is_curator read = code_permission in ('read', 'write') or is_curator
@ -197,11 +206,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
zones_access=is_curator, zones_access=is_curator,
) )
def _get_unit_content( def _get_unit_content(self, team: dict, unit_name: str) -> str:
self,
team: dict,
unit_name: str
) -> str:
permissions = team['units_map'].get(unit_name) permissions = team['units_map'].get(unit_name)
if not permissions: if not permissions:
logger.debug(f'no unit `repo.actions` in team {self.team}') logger.debug(f'no unit `repo.actions` in team {self.team}')
@ -216,23 +221,22 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
return permissions return permissions
def _instance_label(self) -> str: def _instance_label(self) -> str:
return self.instance_id or hashlib.md5( return self.instance_id or hashlib.md5(self.api_url.encode()).hexdigest()
self.api_url.encode()
).hexdigest()
@MethodCache.cache_temporary(duration=60) @MethodCache.cache_temporary(duration=60)
def authenticate( def authenticate(
self, self,
token: str, token: str,
) -> AuthenticationInfo: ) -> AuthenticationInfo:
logger.debug(
logger.debug(f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}') f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}'
)
user_teams = self._get_teams_for_user(token) user_teams = self._get_teams_for_user(token)
logger.debug(f'user_teams: {user_teams}') logger.debug(f'user_teams: {user_teams}')
if self.team not in user_teams: if self.team not in user_teams:
logger.debug(f'{self.team} not in user\'s teams') logger.debug(f"{self.team} not in user's teams")
msg = f'token user is not member of team `{self.team}`' msg = f'token user is not member of team `{self.team}`'
raise RemoteAuthenticationError( raise RemoteAuthenticationError(
status=HTTP_401_UNAUTHORIZED, status=HTTP_401_UNAUTHORIZED,
@ -281,8 +285,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
action_permissions, action_permissions,
), ),
user_id=user_info['email'], user_id=user_info['email'],
incoming_label= incoming_label=f'forgejo-{self._instance_label()}-team-{organization["name"]}-{team["name"]}'
f'forgejo-{self._instance_label()}-team-{organization["name"]}-{team["name"]}' if self.label_type == 'team'
if self.label_type == 'team' else f'forgejo-{self._instance_label()}-user-{user_info["login"]}',
else f'forgejo-{self._instance_label()}-user-{user_info["login"]}',
) )

28
dump_things_service/authenticate.py
View file

@ -1,56 +1,34 @@
from __future__ import annotations from __future__ import annotations
import logging
from itertools import count
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from fastapi import ( from fastapi import (
APIRouter,
Depends,
FastAPI,
HTTPException, HTTPException,
) )
from fastapi_pagination import (
Page,
add_pagination,
paginate,
)
from dump_things_service import ( from dump_things_service import (
HTTP_401_UNAUTHORIZED, HTTP_401_UNAUTHORIZED,
HTTP_404_NOT_FOUND,
HTTP_422_UNPROCESSABLE_CONTENT, abstract_config,
) )
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
check_collection, check_collection,
read_config, read_config,
) )
from dump_things_service.api_key import api_key_header_scheme
from dump_things_service.auth import AuthenticationInfo
from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer
from dump_things_service.exceptions import CurieResolutionError
from dump_things_service.instance_state import get_instance_state from dump_things_service.instance_state import get_instance_state
from dump_things_service.lazy_list import ModifierList
from dump_things_service.utils import ( from dump_things_service.utils import (
authenticate_token, authenticate_token,
check_bounds,
cleaned_json,
wrap_http_exception,
) )
if TYPE_CHECKING: if TYPE_CHECKING:
from pydantic import BaseModel from dump_things_service.auth import AuthenticationInfo
from dump_things_service.backends import StorageBackend from dump_things_service.backends import StorageBackend
from dump_things_service.lazy_list import LazyList
from dump_things_service.store.model_store import _ModelStore from dump_things_service.store.model_store import _ModelStore
def get_store_and_backend( def get_store_and_backend(
collection: str, collection: str,
plain_token: str | None, plain_token: str | None,
) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]: ) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]:
# A token is required # A token is required
if plain_token is None: if plain_token is None:
raise HTTPException( raise HTTPException(

50
dump_things_service/backends/__init__.py
View file

@ -83,12 +83,12 @@ class BackendResultList(LazyList):
@abstractmethod @abstractmethod
def generate_result( def generate_result(
self, self,
index: int, index: int,
iri: str, iri: str,
class_name: str, class_name: str,
sort_key: str, sort_key: str,
private: Any, private: Any,
) -> RecordInfo: ) -> RecordInfo:
""" """
Generate a record info object from the provided parameters. Generate a record info object from the provided parameters.
@ -105,23 +105,21 @@ class BackendResultList(LazyList):
class StorageBackend(metaclass=ABCMeta): class StorageBackend(metaclass=ABCMeta):
def __init__( def __init__(
self, self,
order_by: Iterable[str] | None = None, order_by: Iterable[str] | None = None,
): ):
self.order_by = order_by or ['pid'] self.order_by = order_by or ['pid']
@abstractmethod @abstractmethod
def get_uri( def get_uri(self) -> str:
self
) -> str:
raise NotImplementedError raise NotImplementedError
@abstractmethod @abstractmethod
def add_record( def add_record(
self, self,
iri: str, iri: str,
class_name: str, class_name: str,
json_object: dict, json_object: dict,
): ):
raise NotImplementedError raise NotImplementedError
@ -139,37 +137,37 @@ class StorageBackend(metaclass=ABCMeta):
@abstractmethod @abstractmethod
def remove_record( def remove_record(
self, self,
iri: str, iri: str,
) -> bool: ) -> bool:
raise NotImplementedError raise NotImplementedError
@abstractmethod @abstractmethod
def get_record_by_iri( def get_record_by_iri(
self, self,
iri: str, iri: str,
) -> RecordInfo | None: ) -> RecordInfo | None:
raise NotImplementedError raise NotImplementedError
@abstractmethod @abstractmethod
def get_records_of_classes( def get_records_of_classes(
self, self,
class_names: Iterable[str], class_names: Iterable[str],
pattern: str | None = None, pattern: str | None = None,
) -> BackendResultList: ) -> BackendResultList:
raise NotImplementedError raise NotImplementedError
@abstractmethod @abstractmethod
def get_all_records( def get_all_records(
self, self,
pattern: str | None = None, pattern: str | None = None,
) -> BackendResultList: ) -> BackendResultList:
raise NotImplementedError raise NotImplementedError
def create_sort_key( def create_sort_key(
json_object: dict[str, Any], json_object: dict[str, Any],
order_by: Iterable[str], order_by: Iterable[str],
) -> str: ) -> str:
return '-'.join( return '-'.join(
str(json_object.get(key)) if json_object.get(key) is not None else chr(0x10FFFF) str(json_object.get(key)) if json_object.get(key) is not None else chr(0x10FFFF)

73
dump_things_service/backends/record_dir.py
View file

@ -10,7 +10,6 @@ import logging
from pathlib import Path from pathlib import Path
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Callable,
) )
import yaml import yaml
@ -26,12 +25,12 @@ from dump_things_service.backends import (
from dump_things_service.backends.record_dir_index import RecordDirIndex from dump_things_service.backends.record_dir_index import RecordDirIndex
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Iterable from collections.abc import Callable, Iterable
__all__ = [ __all__ = [
'_RecordDirStore',
'RecordDirStore', 'RecordDirStore',
'_RecordDirStore',
] ]
ignored_files = {'.', '..', config_file_name} ignored_files = {'.', '..', config_file_name}
@ -45,12 +44,12 @@ class RecordDirResultList(BackendResultList):
""" """
def generate_result( def generate_result(
self, self,
_: int, _: int,
iri: str, iri: str,
class_name: str, class_name: str,
sort_key: str, sort_key: str,
path: Path, path: Path,
) -> RecordInfo: ) -> RecordInfo:
""" """
Generate a JSON representation of the record at index `index`. Generate a JSON representation of the record at index `index`.
@ -76,11 +75,11 @@ class _RecordDirStore(StorageBackend):
"""Store records in a directory structure""" """Store records in a directory structure"""
def __init__( def __init__(
self, self,
root: Path, root: Path,
pid_mapping_function: Callable, pid_mapping_function: Callable,
suffix: str, suffix: str,
order_by: Iterable[str] | None = None, order_by: Iterable[str] | None = None,
): ):
super().__init__(order_by=order_by) super().__init__(order_by=order_by)
if not root.is_absolute(): if not root.is_absolute():
@ -91,28 +90,26 @@ class _RecordDirStore(StorageBackend):
self.suffix = suffix self.suffix = suffix
self.index = RecordDirIndex(root, suffix) self.index = RecordDirIndex(root, suffix)
def get_uri( def get_uri(self) -> str:
self
) -> str:
return f'file://{self.root!s}' return f'file://{self.root!s}'
def build_index( def build_index(
self, self,
schema: str, schema: str,
): ):
self.index.rebuild_index(schema, self.order_by) self.index.rebuild_index(schema, self.order_by)
def build_index_if_needed( def build_index_if_needed(
self, self,
schema: str, schema: str,
): ):
self.index.rebuild_if_needed(schema, self.order_by) self.index.rebuild_if_needed(schema, self.order_by)
def add_record( def add_record(
self, self,
iri: str, iri: str,
class_name: str, class_name: str,
json_object: dict, json_object: dict,
): ):
pid = json_object['pid'] pid = json_object['pid']
@ -148,8 +145,8 @@ class _RecordDirStore(StorageBackend):
self.index.add_iri_info(iri, class_name, str(storage_path), sort_string) self.index.add_iri_info(iri, class_name, str(storage_path), sort_string)
def get_record_by_iri( def get_record_by_iri(
self, self,
iri: str, iri: str,
) -> RecordInfo | None: ) -> RecordInfo | None:
index_entry = self.index.get_info_for_iri(iri) index_entry = self.index.get_info_for_iri(iri)
if index_entry is None: if index_entry is None:
@ -165,9 +162,9 @@ class _RecordDirStore(StorageBackend):
) )
def get_records_of_classes( def get_records_of_classes(
self, self,
class_names: list[str], class_names: list[str],
pattern: str | None = None, pattern: str | None = None,
) -> RecordDirResultList: ) -> RecordDirResultList:
return RecordDirResultList().add_info( return RecordDirResultList().add_info(
sorted( sorted(
@ -186,8 +183,8 @@ class _RecordDirStore(StorageBackend):
) )
def get_all_records( def get_all_records(
self, self,
pattern: str | None = None, pattern: str | None = None,
) -> RecordDirResultList: ) -> RecordDirResultList:
return RecordDirResultList().add_info( return RecordDirResultList().add_info(
sorted( sorted(
@ -205,8 +202,8 @@ class _RecordDirStore(StorageBackend):
) )
def remove_record( def remove_record(
self, self,
iri: str, iri: str,
) -> bool: ) -> bool:
index_entry = self.index.get_info_for_iri(iri) index_entry = self.index.get_info_for_iri(iri)
if index_entry is None: if index_entry is None:
@ -226,10 +223,10 @@ _existing_stores = {}
def RecordDirStore( # noqa: N802 def RecordDirStore( # noqa: N802
root: Path, root: Path,
pid_mapping_function: Callable, pid_mapping_function: Callable,
suffix: str, suffix: str,
order_by: Iterable[str] | None = None, order_by: Iterable[str] | None = None,
) -> _RecordDirStore: ) -> _RecordDirStore:
"""Get a record directory store for the given root directory.""" """Get a record directory store for the given root directory."""
existing_store = _existing_stores.get(root) existing_store = _existing_stores.get(root)

62
dump_things_service/backends/record_dir_index.py
View file

@ -65,11 +65,11 @@ class IndexEntry(Base):
class RecordDirIndex: class RecordDirIndex:
def __init__( def __init__(
self, self,
store_dir: Path, store_dir: Path,
suffix: str, suffix: str,
*, *,
echo: bool = False, echo: bool = False,
): ):
if not store_dir.is_absolute(): if not store_dir.is_absolute():
msg = f'Not an absolute path: {store_dir}' msg = f'Not an absolute path: {store_dir}'
@ -91,11 +91,11 @@ class RecordDirIndex:
Base.metadata.create_all(self.engine) Base.metadata.create_all(self.engine)
def add_iri_info( def add_iri_info(
self, self,
iri: str, iri: str,
class_name: str, class_name: str,
path: str, path: str,
sort_key: str, sort_key: str,
): ):
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
self.add_iri_info_with_session( self.add_iri_info_with_session(
@ -107,12 +107,12 @@ class RecordDirIndex:
) )
def add_iri_info_with_session( def add_iri_info_with_session(
self, self,
session: Session, session: Session,
iri: str, iri: str,
class_name: str, class_name: str,
path: str, path: str,
sort_key: str, sort_key: str,
): ):
existing_record = session.query(IndexEntry).filter_by(iri=iri).first() existing_record = session.query(IndexEntry).filter_by(iri=iri).first()
if existing_record: if existing_record:
@ -131,8 +131,8 @@ class RecordDirIndex:
) )
def get_info_for_iri( def get_info_for_iri(
self, self,
iri: str, iri: str,
) -> tuple | None: ) -> tuple | None:
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
statement = select(IndexEntry).filter_by(iri=iri) statement = select(IndexEntry).filter_by(iri=iri)
@ -142,8 +142,8 @@ class RecordDirIndex:
return None return None
def get_info_for_class( def get_info_for_class(
self, self,
class_name: str, class_name: str,
) -> Generator[IndexEntry]: ) -> Generator[IndexEntry]:
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
statement = select(IndexEntry).filter_by(class_name=class_name) statement = select(IndexEntry).filter_by(class_name=class_name)
@ -152,7 +152,7 @@ class RecordDirIndex:
yield row[0] yield row[0]
def get_info_for_all_classes( def get_info_for_all_classes(
self, self,
) -> Generator[IndexEntry]: ) -> Generator[IndexEntry]:
statement = select(IndexEntry) statement = select(IndexEntry)
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
@ -161,8 +161,8 @@ class RecordDirIndex:
yield row[0] yield row[0]
def remove_iri_info( def remove_iri_info(
self, self,
iri: str, iri: str,
) -> bool: ) -> bool:
statement = delete(IndexEntry).where(IndexEntry.iri == iri) statement = delete(IndexEntry).where(IndexEntry.iri == iri)
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
@ -170,9 +170,9 @@ class RecordDirIndex:
return result.rowcount == 1 return result.rowcount == 1
def rebuild_index( def rebuild_index(
self, self,
schema: str, schema: str,
order_by: Iterable[str] | None = None, order_by: Iterable[str] | None = None,
): ):
"""Rebuild the index from the records in the directory.""" """Rebuild the index from the records in the directory."""
lgr.info('Building IRI index for records in %s', self.store_dir) lgr.info('Building IRI index for records in %s', self.store_dir)
@ -223,17 +223,17 @@ class RecordDirIndex:
self.needs_rebuild = False self.needs_rebuild = False
def rebuild_if_needed( def rebuild_if_needed(
self, self,
schema: str, schema: str,
order_by: Iterable[str] | None = None, order_by: Iterable[str] | None = None,
): ):
if self.needs_rebuild: if self.needs_rebuild:
self.rebuild_index(schema=schema, order_by=order_by) self.rebuild_index(schema=schema, order_by=order_by)
self.needs_rebuild = False self.needs_rebuild = False
def _get_class_name( def _get_class_name(
self, self,
path: Path, path: Path,
) -> str: ) -> str:
"""Get the class name from the path.""" """Get the class name from the path."""
rel_path = path.absolute().relative_to(self.store_dir) rel_path = path.absolute().relative_to(self.store_dir)

67
dump_things_service/backends/schema_type_layer.py
View file

@ -34,16 +34,16 @@ if TYPE_CHECKING:
__all__ = [ __all__ = [
'_SchemaTypeLayer',
'SchemaTypeLayer', 'SchemaTypeLayer',
'_SchemaTypeLayer',
] ]
class SchemaTypeLayerResultList(BackendResultList): class SchemaTypeLayerResultList(BackendResultList):
def __init__( def __init__(
self, self,
origin_list: BackendResultList, origin_list: BackendResultList,
schema_model: ModuleType, schema_model: ModuleType,
): ):
super().__init__() super().__init__()
self.schema_model = schema_model self.schema_model = schema_model
@ -51,12 +51,12 @@ class SchemaTypeLayerResultList(BackendResultList):
self.list_info = self.origin_list.list_info self.list_info = self.origin_list.list_info
def generate_result( def generate_result(
self, self,
index: int, index: int,
iri: str, iri: str,
class_name: str, class_name: str,
sort_key: str, sort_key: str,
private: Any, private: Any,
) -> RecordInfo: ) -> RecordInfo:
origin_element = self.origin_list.generate_result( origin_element = self.origin_list.generate_result(
index, iri, class_name, sort_key, private index, iri, class_name, sort_key, private
@ -73,31 +73,28 @@ class _SchemaTypeLayer(StorageBackend):
"""Proxy backend that removes `schema_type` from stored records""" """Proxy backend that removes `schema_type` from stored records"""
def __init__( def __init__(
self, self,
backend: StorageBackend, backend: StorageBackend,
schema: str, schema: str,
): ):
super().__init__() super().__init__()
self.backend = backend self.backend = backend
self.schema_model = get_schema_model_for_schema(schema) self.schema_model = get_schema_model_for_schema(schema)
def get_uri( def get_uri(self) -> str:
self
) -> str:
return self.backend.get_uri() return self.backend.get_uri()
def add_record( def add_record(
self, self,
iri: str, iri: str,
class_name: str, class_name: str,
json_object: dict, json_object: dict,
): ):
# Remove the top level `schema_type` from the JSON object because we # Remove the top level `schema_type` from the JSON object because we
# don't want to store it in the files. We add `schema_type` after # don't want to store it in the files. We add `schema_type` after
# reading the record from disk. The value of `schema_type` is determined # reading the record from disk. The value of `schema_type` is determined
# by the class name of the record, which is stored in the path. # by the class name of the record, which is stored in the path.
if 'schema_type' in json_object: json_object.pop('schema_type', None)
del json_object['schema_type']
self.backend.add_record( self.backend.add_record(
iri=iri, iri=iri,
class_name=class_name, class_name=class_name,
@ -105,14 +102,14 @@ class _SchemaTypeLayer(StorageBackend):
) )
def remove_record( def remove_record(
self, self,
iri: str, iri: str,
) -> bool: ) -> bool:
return self.backend.remove_record(iri=iri) return self.backend.remove_record(iri=iri)
def get_record_by_iri( def get_record_by_iri(
self, self,
iri: str, iri: str,
) -> RecordInfo | None: ) -> RecordInfo | None:
origin_result = self.backend.get_record_by_iri(iri) origin_result = self.backend.get_record_by_iri(iri)
if origin_result and 'schema_type' not in origin_result.json_object: if origin_result and 'schema_type' not in origin_result.json_object:
@ -123,9 +120,9 @@ class _SchemaTypeLayer(StorageBackend):
return origin_result return origin_result
def get_records_of_classes( def get_records_of_classes(
self, self,
class_names: list[str], class_names: list[str],
pattern: str | None = None, pattern: str | None = None,
) -> BackendResultList: ) -> BackendResultList:
return SchemaTypeLayerResultList( return SchemaTypeLayerResultList(
origin_list=self.backend.get_records_of_classes( origin_list=self.backend.get_records_of_classes(
@ -136,8 +133,8 @@ class _SchemaTypeLayer(StorageBackend):
) )
def get_all_records( def get_all_records(
self, self,
pattern: str | None = None, pattern: str | None = None,
) -> BackendResultList: ) -> BackendResultList:
return SchemaTypeLayerResultList( return SchemaTypeLayerResultList(
origin_list=self.backend.get_all_records(pattern), origin_list=self.backend.get_all_records(pattern),
@ -150,8 +147,8 @@ class _SchemaTypeLayer(StorageBackend):
def _get_schema_type( def _get_schema_type(
class_name: str, class_name: str,
schema_module: ModuleType, schema_module: ModuleType,
) -> str: ) -> str:
return getattr(schema_module, class_name).class_class_curie return getattr(schema_module, class_name).class_class_curie
@ -161,8 +158,8 @@ _existing_layers = {}
def SchemaTypeLayer( # noqa: N802 def SchemaTypeLayer( # noqa: N802
backend: StorageBackend, backend: StorageBackend,
schema: str, schema: str,
) -> _SchemaTypeLayer: ) -> _SchemaTypeLayer:
existing_layer, _ = _existing_layers.get(id(backend), (None, None)) existing_layer, _ = _existing_layers.get(id(backend), (None, None))
if not existing_layer: if not existing_layer:

87
dump_things_service/backends/sqlite.py
View file

@ -62,8 +62,8 @@ if TYPE_CHECKING:
__all__ = [ __all__ = [
'_SQLiteBackend',
'SQLiteBackend', 'SQLiteBackend',
'_SQLiteBackend',
] ]
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
@ -88,19 +88,19 @@ class Thing(Base):
class SQLResultList(BackendResultList): class SQLResultList(BackendResultList):
def __init__( def __init__(
self, self,
engine: Any, engine: Any,
): ):
super().__init__() super().__init__()
self.engine = engine self.engine = engine
def generate_result( def generate_result(
self, self,
_: int, _: int,
iri: str, iri: str,
class_name: str, class_name: str,
sort_key: str, sort_key: str,
db_id: int, db_id: int,
) -> RecordInfo: ) -> RecordInfo:
""" """
Generate a JSON representation of the record at index `index`. Generate a JSON representation of the record at index `index`.
@ -124,11 +124,11 @@ class SQLResultList(BackendResultList):
class _SQLiteBackend(StorageBackend): class _SQLiteBackend(StorageBackend):
def __init__( def __init__(
self, self,
db_path: Path, db_path: Path,
*, *,
order_by: Iterable[str] | None = None, order_by: Iterable[str] | None = None,
echo: bool = False, echo: bool = False,
) -> None: ) -> None:
assert db_path.is_absolute(), f'db_path not absolute {db_path}' assert db_path.is_absolute(), f'db_path not absolute {db_path}'
if db_path.exists(): if db_path.exists():
@ -139,9 +139,7 @@ class _SQLiteBackend(StorageBackend):
self.engine = create_engine('sqlite:///' + str(db_path), echo=echo) self.engine = create_engine('sqlite:///' + str(db_path), echo=echo)
Base.metadata.create_all(self.engine) Base.metadata.create_all(self.engine)
def get_uri( def get_uri(self) -> str:
self
) -> str:
return f'sqlite://{self.db_path}' return f'sqlite://{self.db_path}'
def perform_file_name_conversion(self): def perform_file_name_conversion(self):
@ -152,7 +150,9 @@ class _SQLiteBackend(StorageBackend):
logger.info('converting old style name %s', str(old_path)) logger.info('converting old style name %s', str(old_path))
# Create a backup copy # Create a backup copy
old_backup_path = (self.db_path.parent / (old_record_file_name + '.backup')).absolute() old_backup_path = (
self.db_path.parent / (old_record_file_name + '.backup')
).absolute()
logger.info('copying %s to %s', old_path, old_backup_path) logger.info('copying %s to %s', old_path, old_backup_path)
shutil.copyfile(str(old_path), str(old_backup_path)) shutil.copyfile(str(old_path), str(old_backup_path))
@ -161,10 +161,10 @@ class _SQLiteBackend(StorageBackend):
shutil.move(str(old_path), str(self.db_path)) shutil.move(str(old_path), str(self.db_path))
def add_record( def add_record(
self, self,
iri: str, iri: str,
class_name: str, class_name: str,
json_object: dict, json_object: dict,
): ):
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
self._add_record_with_session( self._add_record_with_session(
@ -175,8 +175,8 @@ class _SQLiteBackend(StorageBackend):
) )
def add_records_bulk( def add_records_bulk(
self, self,
record_infos: Iterable[RecordInfo], record_infos: Iterable[RecordInfo],
): ):
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
for record_info in record_infos: for record_info in record_infos:
@ -188,8 +188,8 @@ class _SQLiteBackend(StorageBackend):
) )
def remove_record( def remove_record(
self, self,
iri: str, iri: str,
) -> bool: ) -> bool:
statement = delete(Thing).where(Thing.iri == iri) statement = delete(Thing).where(Thing.iri == iri)
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
@ -197,11 +197,11 @@ class _SQLiteBackend(StorageBackend):
return result.rowcount == 1 return result.rowcount == 1
def _add_record_with_session( def _add_record_with_session(
self, self,
session: Session, session: Session,
iri: str, iri: str,
class_name: str, class_name: str,
json_object: dict, json_object: dict,
): ):
sort_key = create_sort_key(json_object, self.order_by) sort_key = create_sort_key(json_object, self.order_by)
existing_record = session.query(Thing).filter_by(iri=iri).first() existing_record = session.query(Thing).filter_by(iri=iri).first()
@ -220,8 +220,8 @@ class _SQLiteBackend(StorageBackend):
) )
def get_record_by_iri( def get_record_by_iri(
self, self,
iri: str, iri: str,
) -> RecordInfo | None: ) -> RecordInfo | None:
with Session(self.engine) as session, session.begin(): with Session(self.engine) as session, session.begin():
statement = select(Thing).filter_by(iri=iri) statement = select(Thing).filter_by(iri=iri)
@ -236,25 +236,24 @@ class _SQLiteBackend(StorageBackend):
return None return None
def get_records_of_classes( def get_records_of_classes(
self, self,
class_names: Iterable[str], class_names: Iterable[str],
pattern: str | None = None, pattern: str | None = None,
) -> SQLResultList: ) -> SQLResultList:
class_list = ', '.join(f"'{cn}'" for cn in class_names) class_list = ', '.join(f"'{cn}'" for cn in class_names)
if pattern is None: if pattern is None:
statement = text( statement = text(
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id '
'from thing ' 'from thing '
f"where thing.class_name in ({class_list}) " f'where thing.class_name in ({class_list}) '
"ORDER BY thing.sort_key" 'ORDER BY thing.sort_key'
) )
else: else:
statement = text( statement = text(
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id '
'from thing, json_tree(thing.object) ' 'from thing, json_tree(thing.object) '
'where lower(json_tree.value) like lower(:pattern) ' 'where lower(json_tree.value) like lower(:pattern) '
f"and thing.class_name in ({class_list}) " f'and thing.class_name in ({class_list}) '
"and json_tree.type = 'text' ORDER BY thing.sort_key" "and json_tree.type = 'text' ORDER BY thing.sort_key"
) )
@ -271,14 +270,14 @@ class _SQLiteBackend(StorageBackend):
) )
def get_all_records( def get_all_records(
self, self,
pattern: str | None = None, pattern: str | None = None,
) -> SQLResultList: ) -> SQLResultList:
if pattern is None: if pattern is None:
statement = text( statement = text(
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id '
'from thing ' 'from thing '
"ORDER BY thing.sort_key" 'ORDER BY thing.sort_key'
) )
else: else:
statement = text( statement = text(
@ -306,7 +305,7 @@ _existing_sqlite_backends = {}
def SQLiteBackend( # noqa: N802 def SQLiteBackend( # noqa: N802
db_path: Path, *, order_by: Iterable[str] | None = None, echo: bool = False db_path: Path, *, order_by: Iterable[str] | None = None, echo: bool = False
) -> _SQLiteBackend: ) -> _SQLiteBackend:
existing_backend = _existing_sqlite_backends.get(db_path) existing_backend = _existing_sqlite_backends.get(db_path)
if not existing_backend: if not existing_backend:

4
dump_things_service/backends/tests/test_record_dir.py
View file

@ -20,9 +20,7 @@ def test_add_and_delete_record(tmp_path):
record_dir_store.build_index(str(schema_path)) record_dir_store.build_index(str(schema_path))
record_dir_store.add_record( record_dir_store.add_record(
iri=iri, iri=iri, class_name='Object', json_object={'pid': 'some-pid'}
class_name='Object',
json_object={'pid': 'some-pid'}
) )
record = record_dir_store.get_record_by_iri(iri=iri) record = record_dir_store.get_record_by_iri(iri=iri)

262
dump_things_service/collection.py
View file

@ -2,13 +2,19 @@ import logging
import os import os
import shutil import shutil
from pathlib import Path from pathlib import Path
from typing import Any
# This following lines are required for dynamic endpoint generation
from typing import (
Annotated, # noqa: F401 -- used by autogenerated code
Any,
)
from datalad_core.runners import ( from datalad_core.runners import (
call_git_oneline,
CommandError, CommandError,
call_git_oneline,
) )
from fastapi import ( from fastapi import (
Body, # noqa: F401 -- used by autogenerated code
Depends, Depends,
FastAPI, FastAPI,
HTTPException, HTTPException,
@ -24,40 +30,49 @@ from starlette.responses import (
) )
from dump_things_service import ( from dump_things_service import (
Format,
HTTP_400_BAD_REQUEST, HTTP_400_BAD_REQUEST,
HTTP_403_FORBIDDEN, HTTP_403_FORBIDDEN,
HTTP_422_UNPROCESSABLE_CONTENT, HTTP_422_UNPROCESSABLE_CONTENT,
Format,
) )
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
CollectionConfig, CollectionConfig,
Configuration,
ConfigAuthSpec, ConfigAuthSpec,
Configuration,
ForgejoAuthSpec, ForgejoAuthSpec,
RecordDirBackendConfig, RecordDirBackendConfig,
SQLiteBackendConfig, SQLiteBackendConfig,
read_config,
check_collection, check_collection,
read_config,
)
from dump_things_service.api_key import (
api_key_header_scheme,
) )
from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.audit.gitaudit import GitAuditBackend
from dump_things_service.auth.config import ConfigAuthenticationSource from dump_things_service.auth.config import ConfigAuthenticationSource
from dump_things_service.auth.forgejo import ForgejoAuthenticationSource from dump_things_service.auth.forgejo import ForgejoAuthenticationSource
from dump_things_service.backends.record_dir_index import index_file_name from dump_things_service.backends.record_dir_index import index_file_name
from dump_things_service.backends.sqlite import record_file_name as sqlite_db_filename from dump_things_service.backends.sqlite import record_file_name as sqlite_db_filename
from dump_things_service.converter import FormatConverter
from dump_things_service.curated import (
store_curated_record, # noqa: F401 -- used by autogenerated code
)
from dump_things_service.exceptions import (
ConfigCollisionError,
ConfigError,
CurieResolutionError,
)
from dump_things_service.incoming import (
store_incoming_record, # noqa: F401 -- used by autogenerated code
)
from dump_things_service.instance_state import ( from dump_things_service.instance_state import (
InstanceState, InstanceState,
InstanceStateCollectionInfo, InstanceStateCollectionInfo,
get_record_dir_config,
get_instance_state, get_instance_state,
get_record_dir_config,
get_schema_info, get_schema_info,
record_dir_config_file_name, record_dir_config_file_name,
) )
from dump_things_service.converter import FormatConverter
from dump_things_service.exceptions import (
ConfigError,
ConfigCollisionError,
CurieResolutionError,
)
from dump_things_service.model import get_model_for_schema from dump_things_service.model import get_model_for_schema
from dump_things_service.utils import ( from dump_things_service.utils import (
combine_ttl, combine_ttl,
@ -67,16 +82,9 @@ from dump_things_service.utils import (
var_escape, var_escape,
wrap_http_exception, wrap_http_exception,
) )
from dump_things_service.validate import (
validate_record, # noqa: F401 -- used by autogenerated code
# This following lines are required for dynamic endpoint generation )
from typing import Annotated # noqa 401 -- used by autogenerated code
from fastapi import Body # noqa 401 -- used by autogenerated code
from dump_things_service.api_key import api_key_header_scheme # noqa 401 -- used by autogenerated code
from dump_things_service.curated import store_curated_record # noqa 401 -- used by autogenerated code
from dump_things_service.incoming import store_incoming_record # noqa 401 -- used by autogenerated code
from dump_things_service.validate import validate_record # noqa 401 -- used by autogenerated code
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
@ -135,9 +143,9 @@ async def {name}(
def create_collection( def create_collection(
instance_state: InstanceState, instance_state: InstanceState,
configuration: Configuration, configuration: Configuration,
collection_name: str, collection_name: str,
): ):
"""Create a collection instance as specified by `collection_configuration` """Create a collection instance as specified by `collection_configuration`
@ -191,7 +199,7 @@ def create_collection(
audit_path.mkdir(parents=True) audit_path.mkdir(parents=True)
created_directories.append(audit_path) created_directories.append(audit_path)
except ConfigError as e: except ConfigError:
# Delete all directories that were created in this # Delete all directories that were created in this
for directory in created_directories: for directory in created_directories:
shutil.rmtree(directory) shutil.rmtree(directory)
@ -222,7 +230,7 @@ def create_collection(
active_classes -= set(collection_configuration.ignore_classes) active_classes -= set(collection_configuration.ignore_classes)
instance_state.collections[collection_name] = InstanceStateCollectionInfo( instance_state.collections[collection_name] = InstanceStateCollectionInfo(
active_classes=active_classes, active_classes=active_classes,
tag_info=dict(), tag_info={},
) )
# Create a validator for the collection # Create a validator for the collection
@ -262,10 +270,10 @@ def create_collection(
def create_authentication_source( def create_authentication_source(
abstract_configuration: Configuration, abstract_configuration: Configuration,
collection_name: str, collection_name: str,
authentication_spec: ConfigAuthSpec | ForgejoAuthSpec, authentication_spec: ConfigAuthSpec | ForgejoAuthSpec,
instance_state: InstanceState, instance_state: InstanceState,
): ):
if collection_name not in instance_state.auth_sources: if collection_name not in instance_state.auth_sources:
instance_state.auth_sources[collection_name] = [] instance_state.auth_sources[collection_name] = []
@ -293,15 +301,16 @@ def create_authentication_source(
def write_record_dir_config( def write_record_dir_config(
path: Path, path: Path,
backend_config: RecordDirBackendConfig, backend_config: RecordDirBackendConfig,
schema: str, schema: str,
): ):
assert isinstance(backend_config, RecordDirBackendConfig) assert isinstance(backend_config, RecordDirBackendConfig)
record_dir_config_file_path = path / record_dir_config_file_name record_dir_config_file_path = path / record_dir_config_file_name
if not record_dir_config_file_path.exists(): if not record_dir_config_file_path.exists():
record_dir_config_file_path.write_text(f"""# RecordDir Config record_dir_config_file_path.write_text(
f"""# RecordDir Config
type: records type: records
version: 1 version: 1
schema: {schema} schema: {schema}
@ -312,9 +321,9 @@ idfx: {backend_config.mapping_method}
def check_store_compatibility( def check_store_compatibility(
store_path: Path, store_path: Path,
backend_config: RecordDirBackendConfig | SQLiteBackendConfig, backend_config: RecordDirBackendConfig | SQLiteBackendConfig,
schema: str, schema: str,
): ):
"""Check if an existing store is compatible with the specs in `backend_config` """Check if an existing store is compatible with the specs in `backend_config`
@ -336,24 +345,24 @@ def check_store_compatibility(
def check_record_dir_compatibility( def check_record_dir_compatibility(
store_path: Path, store_path: Path,
backend_config: RecordDirBackendConfig, backend_config: RecordDirBackendConfig,
schema: str, schema: str,
): ):
# Non-existing or empty record_dir-directories are compatible # Non-existing or empty record_dir-directories are compatible
if not store_path.exists(): if not store_path.exists():
return return
# A record_dir-directory is considered to be empty, if it contains no # A record_dir-directory is considered to be empty, if it contains no
# files or only an record_dir-index file # files or only an record_dir-index file
files_in_dir = tuple(map(lambda dir_entry: dir_entry.name, os.scandir(store_path))) files_in_dir = tuple(dir_entry.name for dir_entry in os.scandir(store_path))
if files_in_dir in ((), (index_file_name,)): if files_in_dir in ((), (index_file_name,)):
return return
record_dir_config = get_record_dir_config(store_path) record_dir_config = get_record_dir_config(store_path)
if record_dir_config.schema_location != schema: if record_dir_config.schema_location != schema:
raise ConfigCollisionError(f"Existing backend uses a different schema: '{record_dir_config.schema_location}'") msg = f"Existing backend uses a different schema: '{record_dir_config.schema_location}'"
raise ConfigCollisionError(msg)
stored_mapping_method = record_dir_config.idfx.value stored_mapping_method = record_dir_config.idfx.value
if stored_mapping_method != backend_config.mapping_method: if stored_mapping_method != backend_config.mapping_method:
@ -363,16 +372,16 @@ def check_record_dir_compatibility(
def check_sqlite_compatibility( def check_sqlite_compatibility(
store_path: Path, store_path: Path,
): ):
sqlite_db_path = Path(store_path / sqlite_db_filename) sqlite_db_path = Path(store_path / sqlite_db_filename)
if not sqlite_db_path.exists(): if not sqlite_db_path.exists():
raise ConfigError('No sqlite database found in existing store') msg = 'No sqlite database found in existing store'
return raise ConfigError(msg)
def check_git_audit_compatibility( def check_git_audit_compatibility(
audit_path: Path, audit_path: Path,
): ):
"""Check if an existing audit path is compatible with a git audit store """Check if an existing audit path is compatible with a git audit store
@ -394,26 +403,28 @@ def check_git_audit_compatibility(
force_c_locale=True, force_c_locale=True,
) )
except CommandError as ce: except CommandError as ce:
raise ConfigError(f'No git repository in gitaudit-path: {audit_path}') from ce msg = f'No git repository in gitaudit-path: {audit_path}'
raise ConfigError(msg) from ce
if result.strip().lower() != 'true': if result.strip().lower() != 'true':
raise ConfigError(f'No bare git repository in gitaudit-path: {audit_path}') msg = f'No bare git repository in gitaudit-path: {audit_path}'
raise ConfigError(msg)
return return
def create_endpoint( def create_endpoint(
operation_name: str, operation_name: str,
operation_path: str, operation_path: str,
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
collection_config: CollectionConfig, collection_config: CollectionConfig,
template: str, template: str,
handler: str, handler: str,
tag_group: str, tag_group: str,
tag_name: str, tag_name: str,
app: FastAPI, app: FastAPI,
): ):
logger.info( logger.info(
f'Creating %s-endpoints for collection: "%s"', 'Creating %s-endpoints for collection: "%s"',
operation_name, operation_name,
collection_name, collection_name,
) )
@ -421,12 +432,16 @@ def create_endpoint(
instance_state.collections[collection_name].tag_info[tag_group] = tag_name instance_state.collections[collection_name].tag_info[tag_group] = tag_name
# TODO: get schema_info from instance_state!? # TODO: get schema_info from instance_state!?
model, classes, model_var_name = get_model_for_schema(collection_config.schema_location) model, _classes, model_var_name = get_model_for_schema(
collection_config.schema_location
)
globals()[model_var_name] = model globals()[model_var_name] = model
active_classes = instance_state.collections[collection_name].active_classes active_classes = instance_state.collections[collection_name].active_classes
for class_name in active_classes: for class_name in active_classes:
endpoint_name = f'_endpoint_{var_escape(collection_name)}_{operation_name}_{class_name}' endpoint_name = (
f'_endpoint_{var_escape(collection_name)}_{operation_name}_{class_name}'
)
endpoint_source = template.format( endpoint_source = template.format(
name=endpoint_name, name=endpoint_name,
model_var_name=model_var_name, model_var_name=model_var_name,
@ -435,7 +450,7 @@ def create_endpoint(
info=f"'{operation_name} {collection_name}/{class_name} objects'", info=f"'{operation_name} {collection_name}/{class_name} objects'",
handler=handler, handler=handler,
) )
exec(endpoint_source, globals()) # noqa S102 exec(endpoint_source, globals()) # noqa: S102
# Create an API route for the endpoint # Create an API route for the endpoint
app.add_api_route( app.add_api_route(
@ -444,7 +459,7 @@ def create_endpoint(
methods=['POST'], methods=['POST'],
name=f'{operation_name} "{class_name}" object (schema: {model.linkml_meta["id"]})', name=f'{operation_name} "{class_name}" object (schema: {model.linkml_meta["id"]})',
response_model=None, response_model=None,
tags=[tag_name] tags=[tag_name],
) )
logger.info( logger.info(
@ -455,23 +470,51 @@ def create_endpoint(
def create_endpoints_for_collection( def create_endpoints_for_collection(
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
collection_config: CollectionConfig, collection_config: CollectionConfig,
app: FastAPI, app: FastAPI,
): ):
for ( for (
operation_name, operation_name,
operation_path, operation_path,
template, template,
handler, handler,
tag_group, tag_group,
tag_name, tag_name,
) in ( ) in (
('store', 'record', _endpoint_template, 'store_record', 'write', f'Write records to collection "{collection_name}"'), (
('validate', 'validate/record', _endpoint_template, 'validate_record', 'validate', f'Validate records for collection "{collection_name}"'), 'store',
('curated', 'curated/record', _endpoint_curated_template, 'store_curated_record', 'curated_write', f'Curated area: store records in curated area of collection "{collection_name}"'), 'record',
('incoming', 'incoming/{label}/record', _endpoint_incoming_template, 'store_incoming_record', 'incoming_write', f'Incoming area: store records in incoming area "{{label}}" of collection "{collection_name}"'), _endpoint_template,
'store_record',
'write',
f'Write records to collection "{collection_name}"',
),
(
'validate',
'validate/record',
_endpoint_template,
'validate_record',
'validate',
f'Validate records for collection "{collection_name}"',
),
(
'curated',
'curated/record',
_endpoint_curated_template,
'store_curated_record',
'curated_write',
f'Curated area: store records in curated area of collection "{collection_name}"',
),
(
'incoming',
'incoming/{label}/record',
_endpoint_incoming_template,
'store_incoming_record',
'incoming_write',
f'Incoming area: store records in incoming area "{{label}}" of collection "{collection_name}"',
),
): ):
create_endpoint( create_endpoint(
operation_name=operation_name, operation_name=operation_name,
@ -488,17 +531,16 @@ def create_endpoints_for_collection(
def delete_endpoints_for_collection( def delete_endpoints_for_collection(
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
): ):
active_classes = instance_state.collections[collection_name].active_classes active_classes = instance_state.collections[collection_name].active_classes
for operation_path in ( for operation_path in (
'record', 'record',
'validate/record', 'validate/record',
'curated/record', 'curated/record',
'incoming/{label}/record' 'incoming/{label}/record',
): ):
delete_endpoint( delete_endpoint(
collection_name=collection_name, collection_name=collection_name,
@ -509,17 +551,17 @@ def delete_endpoints_for_collection(
def delete_endpoint( def delete_endpoint(
collection_name: str, collection_name: str,
active_classes: set[str], active_classes: set[str],
operation_path: str, operation_path: str,
app: FastAPI, app: FastAPI,
): ):
from fastapi.routing import _IncludedRouter from fastapi.routing import _IncludedRouter
remove_paths_set = set( remove_paths_set = {
f'/{collection_name}/{operation_path}/{class_name}' f'/{collection_name}/{operation_path}/{class_name}'
for class_name in active_classes for class_name in active_classes
) }
remove_indices = [ remove_indices = [
index index
@ -532,13 +574,13 @@ def delete_endpoint(
def store_record( def store_record(
collection: str, collection: str,
data: BaseModel | str, data: BaseModel | str,
class_name: str, class_name: str,
model: Any, model: Any,
input_format: Format, input_format: Format,
add_submission_tag: bool, add_submission_tag: bool,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
) -> JSONResponse | PlainTextResponse: ) -> JSONResponse | PlainTextResponse:
if input_format == Format.json and isinstance(data, str): if input_format == Format.json and isinstance(data, str):
raise HTTPException( raise HTTPException(
@ -584,18 +626,32 @@ def store_record(
) )
if input_format == Format.ttl: if input_format == Format.ttl:
with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Conversion error'): with wrap_http_exception(
ValueError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Conversion error',
):
json_object = FormatConverter( json_object = FormatConverter(
abstract_config.collections[collection].schema_location, abstract_config.collections[collection].schema_location,
input_format=Format.ttl, input_format=Format.ttl,
output_format=Format.json, output_format=Format.json,
).convert(data, class_name) ).convert(data, class_name)
with wrap_http_exception(ValidationError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): with wrap_http_exception(
record = TypeAdapter(getattr(model, class_name)).validate_python(json_object) ValidationError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Validation error',
):
record = TypeAdapter(getattr(model, class_name)).validate_python(
json_object
)
else: else:
record = data record = data
with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): with wrap_http_exception(
ValueError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Validation error',
):
instance_state.validators[collection].validate(record) instance_state.validators[collection].validate(record)
with wrap_http_exception(CurieResolutionError): with wrap_http_exception(CurieResolutionError):

62
dump_things_service/collection_endpoints.py
View file

@ -3,7 +3,7 @@ from pathlib import (
Path, Path,
PurePosixPath, PurePosixPath,
) )
from typing import Literal from typing import Annotated, Literal
from urllib.parse import quote from urllib.parse import quote
from fastapi import ( from fastapi import (
@ -22,19 +22,19 @@ from dump_things_service import (
reserved_collection_names, reserved_collection_names,
) )
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
Configuration,
CollectionConfig, CollectionConfig,
Configuration,
get_config,
get_token_permissions,
store_config, store_config,
get_config, get_token_permissions,
) )
from dump_things_service.admin import authenticate_admin from dump_things_service.admin import authenticate_admin
from dump_things_service.api_key import api_key_header_scheme from dump_things_service.api_key import api_key_header_scheme
from dump_things_service.instance_state import get_instance_state, InstanceState
from dump_things_service.manifest import manifest_configuration
from dump_things_service.exceptions import ConfigError from dump_things_service.exceptions import ConfigError
from dump_things_service.instance_state import InstanceState, get_instance_state
from dump_things_service.manifest import manifest_configuration
from dump_things_service.utils import wrap_http_exception from dump_things_service.utils import wrap_http_exception
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
router = APIRouter() router = APIRouter()
@ -69,9 +69,9 @@ class CollectionRequest(CollectionConfig):
status_code=HTTP_201_CREATED, status_code=HTTP_201_CREATED,
) )
async def create_collection( async def create_collection(
response: Response, response: Response,
body: CollectionRequest, body: CollectionRequest,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
await create_or_replace_collection(body, api_key, allow_replace=False) await create_or_replace_collection(body, api_key, allow_replace=False)
response.headers['Location'] = f'/collections/{quote(body.name)}' response.headers['Location'] = f'/collections/{quote(body.name)}'
@ -84,20 +84,19 @@ async def create_collection(
status_code=HTTP_201_CREATED, status_code=HTTP_201_CREATED,
) )
async def replace_collection( async def replace_collection(
response: Response, response: Response,
body: CollectionRequest, body: CollectionRequest,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
await create_or_replace_collection(body, api_key, allow_replace=True) await create_or_replace_collection(body, api_key, allow_replace=True)
response.headers['Location'] = f'/collections/{quote(body.name)}' response.headers['Location'] = f'/collections/{quote(body.name)}'
async def create_or_replace_collection( async def create_or_replace_collection(
body: CollectionRequest, body: CollectionRequest,
api_key: str, api_key: str,
allow_replace: bool, allow_replace: bool,
): ):
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -165,9 +164,8 @@ async def create_or_replace_collection(
name='Get existing collections', name='Get existing collections',
) )
async def get_collections( async def get_collections(
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> list[CollectionRequest]: ) -> list[CollectionRequest]:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -177,7 +175,7 @@ async def get_collections(
CollectionRequest( CollectionRequest(
**{ **{
'name': collection_name, 'name': collection_name,
**collection_info.model_dump(mode='json', by_alias=True) **collection_info.model_dump(mode='json', by_alias=True),
} }
) )
for collection_name, collection_info in abstract_config.collections.items() for collection_name, collection_info in abstract_config.collections.items()
@ -190,10 +188,9 @@ async def get_collections(
name='Get existing collection by name', name='Get existing collection by name',
) )
async def get_collection_with_name( async def get_collection_with_name(
collection_name: str, collection_name: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> CollectionConfig: ) -> CollectionConfig:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -214,10 +211,9 @@ async def get_collection_with_name(
name='Delete collection with name', name='Delete collection with name',
) )
async def delete_collection( async def delete_collection(
collection_name: str, collection_name: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -245,14 +241,16 @@ async def delete_collection(
def ensure_unique_directory( def ensure_unique_directory(
abstract_config: Configuration, abstract_config: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
existing_dir: PurePosixPath, existing_dir: PurePosixPath,
): ):
abs_existing_dir = (instance_state.store_path / Path(existing_dir)).absolute() abs_existing_dir = (instance_state.store_path / Path(existing_dir)).absolute()
for collection_name, collection_config in abstract_config.collections.items(): for collection_name, collection_config in abstract_config.collections.items():
for collection_dir in collection_config.curated, collection_config.incoming: for collection_dir in collection_config.curated, collection_config.incoming:
abs_collection_dir = (instance_state.store_path / Path(collection_dir)).absolute() abs_collection_dir = (
instance_state.store_path / Path(collection_dir)
).absolute()
if abs_collection_dir == abs_existing_dir: if abs_collection_dir == abs_existing_dir:
raise HTTPException( raise HTTPException(
status_code=HTTP_409_CONFLICT, status_code=HTTP_409_CONFLICT,
@ -261,8 +259,8 @@ def ensure_unique_directory(
def validate_incoming_paths( def validate_incoming_paths(
abstract_config: Configuration, abstract_config: Configuration,
collection_request: CollectionRequest, collection_request: CollectionRequest,
): ):
for token_name, token_info in abstract_config.tokens.items(): for token_name, token_info in abstract_config.tokens.items():
token_collection_info = token_info.collections.get(collection_request.name) token_collection_info = token_info.collections.get(collection_request.name)
@ -273,7 +271,7 @@ def validate_incoming_paths(
detail = ( detail = (
f"Cannot add collection '{collection_request.name}' without " f"Cannot add collection '{collection_request.name}' without "
f"`incoming` path, because at least token '{token_name}' " f"`incoming` path, because at least token '{token_name}' "
f" has write access to the collection" f' has write access to the collection'
) )
raise HTTPException( raise HTTPException(
status_code=HTTP_406_NOT_ACCEPTABLE, status_code=HTTP_406_NOT_ACCEPTABLE,

30
dump_things_service/commands/check_pids.py
View file

@ -2,8 +2,8 @@ from __future__ import annotations
import sys import sys
from argparse import ArgumentParser from argparse import ArgumentParser
from collections.abc import Iterable
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING
from fastapi import FastAPI from fastapi import FastAPI
@ -16,16 +16,20 @@ from dump_things_service.backends.sqlite import _SQLiteBackend
from dump_things_service.exceptions import CurieResolutionError from dump_things_service.exceptions import CurieResolutionError
from dump_things_service.instance_state import create_instance_state from dump_things_service.instance_state import create_instance_state
from dump_things_service.manifest import manifest_configuration from dump_things_service.manifest import manifest_configuration
from dump_things_service.store.model_store import _ModelStore
from dump_things_service.utils import ( from dump_things_service.utils import (
create_token_store, create_token_store,
get_on_disk_labels, get_on_disk_labels,
) )
if TYPE_CHECKING:
from collections.abc import Iterable
from dump_things_service.store.model_store import _ModelStore
parser = ArgumentParser( parser = ArgumentParser(
prog='Check pids for resolvability', prog='Check pids for resolvability',
description='This command checks for pids that are in CURIE format and ' description='This command checks for pids that are in CURIE format and '
'cannot be resolved.', 'cannot be resolved.',
) )
parser.add_argument( parser.add_argument(
'store', 'store',
@ -33,19 +37,7 @@ parser.add_argument(
) )
def show_backend(model_store: _ModelStore): def check_pids_in_stores(stores: Iterable[_ModelStore]) -> int:
backend = model_store.backend
if isinstance(backend, _SchemaTypeLayer):
backend = backend.backend
if isinstance(backend, _SQLiteBackend):
print(f'Checking: {backend.db_path}', file=sys.stderr)
else:
print(f'Checking: {backend.root}', file=sys.stderr)
def check_pids_in_stores(
stores: Iterable[_ModelStore]
) -> int:
result = 0 result = 0
for store in stores: for store in stores:
print('checking', store.get_uri(), file=sys.stderr) print('checking', store.get_uri(), file=sys.stderr)
@ -55,13 +47,11 @@ def check_pids_in_stores(
store.pid_to_iri(pid) store.pid_to_iri(pid)
except CurieResolutionError: except CurieResolutionError:
result += 1 result += 1
print(pid, store.get_uri())
return result return result
def check_pids( def check_pids(
store_path: Path, store_path: Path,
): ):
abstract_config = read_config(store_path) abstract_config = read_config(store_path)
instance_state = create_instance_state( instance_state = create_instance_state(
@ -94,7 +84,7 @@ def check_pids(
abstract_config, abstract_config,
instance_state, instance_state,
collection, collection,
instance_state.store_path / collection_info.incoming / label instance_state.store_path / collection_info.incoming / label,
) )
for label in all_labels for label in all_labels
] ]

2
dump_things_service/commands/copy_store.py
View file

@ -5,6 +5,7 @@ from argparse import ArgumentParser
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from dump_things_service.abstract_config import get_backend_and_extension
from dump_things_service.backends.record_dir import ( from dump_things_service.backends.record_dir import (
RecordDirStore, RecordDirStore,
_RecordDirStore, _RecordDirStore,
@ -17,7 +18,6 @@ from dump_things_service.backends.sqlite import (
from dump_things_service.backends.sqlite import ( from dump_things_service.backends.sqlite import (
record_file_name as sqlite_record_file_name, record_file_name as sqlite_record_file_name,
) )
from dump_things_service.abstract_config import get_backend_and_extension
if TYPE_CHECKING: if TYPE_CHECKING:
from dump_things_service.backends import StorageBackend from dump_things_service.backends import StorageBackend

15
dump_things_service/commands/create_merged_schema.py
View file

@ -5,21 +5,16 @@ import yaml
from linkml_runtime.utils.schemaview import SchemaView from linkml_runtime.utils.schemaview import SchemaView
# Patch linkml # Patch linkml
from dump_things_service.patches import enabled # noqa F401 -- patches LinkML from dump_things_service.patches import enabled # noqa: F401 -- patches LinkML
parser = ArgumentParser( parser = ArgumentParser(
prog='Create a static schema with all imported schemas integrated', prog='Create a static schema with all imported schemas integrated',
) )
parser.add_argument( parser.add_argument('schema', help='File containing a schema definition')
'schema',
help='File containing a schema definition'
)
def update_uris_for_elements( def update_uris_for_elements(
all_elements: dict, all_elements: dict, attribute_name: str, prefix_index: dict
attribute_name: str,
prefix_index: dict
): ):
for name, info in all_elements.items(): for name, info in all_elements.items():
uri = getattr(info, attribute_name) uri = getattr(info, attribute_name)
@ -32,7 +27,7 @@ def update_uris_for_elements(
def update_uris(schema_view: SchemaView): def update_uris(schema_view: SchemaView):
""" Update element-defining URIs to the original element source """Update element-defining URIs to the original element source
Element-defining URIs (e.g., slot_uri, class_uri) are by default set to Element-defining URIs (e.g., slot_uri, class_uri) are by default set to
the schema in which the element is defined. In this case, that would be the the schema in which the element is defined. In this case, that would be the
@ -68,7 +63,7 @@ def main():
Dumper=yaml.SafeDumper, Dumper=yaml.SafeDumper,
allow_unicode=True, allow_unicode=True,
sort_keys=False, sort_keys=False,
) )
print(text) print(text)
return 0 return 0

59
dump_things_service/commands/download_config.py
View file

@ -8,37 +8,38 @@ from argparse import ArgumentParser
import requests import requests
import yaml import yaml
parser = ArgumentParser( parser = ArgumentParser(
prog='Download a complete configuration of a running service', prog='Download a complete configuration of a running service',
description='Read a configuration from dump-things endpoints and create a ' description='Read a configuration from dump-things endpoints and create a '
'configuration-file that can be possibly modified and uploaded ' 'configuration-file that can be possibly modified and uploaded '
'to a running service by dump-things-upload-config.' 'to a running service by dump-things-upload-config.'
' ' ' '
'An admin token has to be provided in the environment variable ' 'An admin token has to be provided in the environment variable '
'`DTS_ADMIN_TOKEN`.', '`DTS_ADMIN_TOKEN`.',
) )
parser.add_argument( parser.add_argument(
'server_api', 'server_api',
help='The base URL of the server API.', help='The base URL of the server API.',
) )
parser.add_argument( parser.add_argument(
'--entities', '-e', '--entities',
'-e',
action='append', action='append',
choices=['admin_tokens', 'collections', 'tokens'], choices=['admin_tokens', 'collections', 'tokens'],
help='Specify for which entities the configuration should be downloaded. ' help='Specify for which entities the configuration should be downloaded. '
' Possible values are `admin_tokens`, `collections`, or `tokens` ' ' Possible values are `admin_tokens`, `collections`, or `tokens` '
'(repeat to download configuration for more than one entity). If this ' '(repeat to download configuration for more than one entity). If this '
'option is not provided, configurations for all entities will be ' 'option is not provided, configurations for all entities will be '
'downloaded.' 'downloaded.',
) )
parser.add_argument( parser.add_argument(
'--format', '-f', '--format',
'-f',
nargs='?', nargs='?',
default='yaml', default='yaml',
choices=['json', 'yaml'], choices=['json', 'yaml'],
help='Specify the format of the output. Possible values are `json` ' help='Specify the format of the output. Possible values are `json` '
'and `yaml` (the default is `yaml`).' 'and `yaml` (the default is `yaml`).',
) )
@ -86,11 +87,10 @@ def main():
def get_configuration( def get_configuration(
api_url: str, api_url: str,
admin_token: str, admin_token: str,
entities: list[str], entities: list[str],
) -> dict: ) -> dict:
result = {} result = {}
if 'collections' in entities: if 'collections' in entities:
@ -110,12 +110,13 @@ def get_configuration(
def list_to_dict_on_key( def list_to_dict_on_key(
elements: list[dict], elements: list[dict],
extract_key: str, extract_key: str,
) -> dict: ) -> dict:
return { return {
element[extract_key]: { element[extract_key]: {
element_key: value for element_key, value in element.items() element_key: value
for element_key, value in element.items()
if element_key != extract_key if element_key != extract_key
} }
for element in elements for element in elements
@ -123,8 +124,8 @@ def list_to_dict_on_key(
def get_tokens( def get_tokens(
api_url: str, api_url: str,
admin_token: str, admin_token: str,
) -> dict: ) -> dict:
token_list = _get_data( token_list = _get_data(
url=api_url + '/tokens', url=api_url + '/tokens',
@ -135,8 +136,8 @@ def get_tokens(
def get_collections( def get_collections(
api_url: str, api_url: str,
admin_token: str, admin_token: str,
) -> dict: ) -> dict:
collection_list = _get_data( collection_list = _get_data(
url=api_url + '/collections', url=api_url + '/collections',
@ -147,8 +148,8 @@ def get_collections(
def get_admin_tokens( def get_admin_tokens(
api_url: str, api_url: str,
admin_token: str, admin_token: str,
) -> dict: ) -> dict:
admin_token_list = _get_data( admin_token_list = _get_data(
url=api_url + '/admin_tokens', url=api_url + '/admin_tokens',
@ -164,9 +165,9 @@ def get_admin_tokens(
def _get_data( def _get_data(
url: str, url: str,
token: str, token: str,
content_class: str, content_class: str,
) -> list: ) -> list:
result = requests.get(url, headers={'x-dumpthings-token': token}) result = requests.get(url, headers={'x-dumpthings-token': token})
if result.status_code >= 300: if result.status_code >= 300:

6
dump_things_service/commands/gitaudit_rebuild_index.py
View file

@ -6,14 +6,12 @@ from pathlib import Path
from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.audit.gitaudit import GitAuditBackend
parser = ArgumentParser( parser = ArgumentParser(
prog='Rebuild the index of a `gitaudit`-database', prog='Rebuild the index of a `gitaudit`-database',
description='This command rebuilds the index of a `gitaudit`-database.' description='This command rebuilds the index of a `gitaudit`-database.',
) )
parser.add_argument( parser.add_argument(
'audit_store', 'audit_store', help='The directory in which the `gitaudit`-database is located.'
help='The directory in which the `gitaudit`-database is located.'
) )

9
dump_things_service/commands/gitaudit_report.py
View file

@ -8,12 +8,11 @@ from pathlib import Path
from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.audit.gitaudit import GitAuditBackend
parser = ArgumentParser( parser = ArgumentParser(
prog='Report audit information for a PID', prog='Report audit information for a PID',
description='Report the audit information that was stored for a specific ' description='Report the audit information that was stored for a specific '
'PID. For every change to a record the tool will report: ' 'PID. For every change to a record the tool will report: '
'time stamp, user ID, diff, and the resulting record.', 'time stamp, user ID, diff, and the resulting record.',
) )
parser.add_argument( parser.add_argument(
'audit_store', 'audit_store',
@ -22,8 +21,8 @@ parser.add_argument(
parser.add_argument( parser.add_argument(
'pid', 'pid',
help='Regex pattern that identifies PIDs of the record for which audit ' help='Regex pattern that identifies PIDs of the record for which audit '
'information should be reported ' 'information should be reported '
'(to see all audit log entries, specify ".*").', '(to see all audit log entries, specify ".*").',
) )

8
dump_things_service/commands/hash_token.py
View file

@ -5,12 +5,11 @@ from argparse import ArgumentParser
from dump_things_service.abstract_config import hash_token_representation from dump_things_service.abstract_config import hash_token_representation
parser = ArgumentParser( parser = ArgumentParser(
prog='Hash a plain text token to create a hashed token in a dump-things server', prog='Hash a plain text token to create a hashed token in a dump-things server',
description='Hash a token and print the calculated hash value. The hash value ' description='Hash a token and print the calculated hash value. The hash value '
'can be used to create a hashed token via the `/tokens`-endpoint ' 'can be used to create a hashed token via the `/tokens`-endpoint '
'of a dump-things-server.', 'of a dump-things-server.',
) )
parser.add_argument( parser.add_argument(
'token', 'token',
@ -23,12 +22,13 @@ def main():
arguments = parser.parse_args() arguments = parser.parse_args()
token = arguments.token.strip() token = arguments.token.strip()
if any(map(lambda s: s.isspace(), token)): if any(s.isspace() for s in token):
print('Whitespace are not allowed in token', file=sys.stderr, flush=True) print('Whitespace are not allowed in token', file=sys.stderr, flush=True)
return 1 return 1
print(hash_token_representation(token)) print(hash_token_representation(token))
return 0 return 0
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())

3
dump_things_service/commands/rebuild_index.py
View file

@ -7,9 +7,8 @@ from pathlib import Path
import yaml import yaml
from dump_things_service import config_file_name from dump_things_service import config_file_name
from dump_things_service.backends.record_dir_index import RecordDirIndex
from dump_things_service.abstract_config import RecordDirConfigFileContent from dump_things_service.abstract_config import RecordDirConfigFileContent
from dump_things_service.backends.record_dir_index import RecordDirIndex
parser = ArgumentParser( parser = ArgumentParser(
prog='Rebuild the index of a `record_dir`-store', prog='Rebuild the index of a `record_dir`-store',

129
dump_things_service/commands/upload_config.py
View file

@ -12,52 +12,52 @@ import yaml
from dump_things_service.instance_state import get_record_dir_config from dump_things_service.instance_state import get_record_dir_config
parser = ArgumentParser( parser = ArgumentParser(
prog='Establish a configuration in a running service', prog='Establish a configuration in a running service',
description='Read a configuration from a dump-things configuration-file ' description='Read a configuration from a dump-things configuration-file '
'and instantiate its elements on a running server. Objects that ' 'and instantiate its elements on a running server. Objects that '
'already exist on the server are left unchanged. ' 'already exist on the server are left unchanged. '
' ' ' '
'An admin token has to be provided in the environment variable ' 'An admin token has to be provided in the environment variable '
'`DTS_ADMIN_TOKEN`.', '`DTS_ADMIN_TOKEN`.',
) )
parser.add_argument( parser.add_argument(
'config_file', 'config_file',
help='The path to the config file', help='The path to the config file',
) )
parser.add_argument( parser.add_argument(
'--format', '-f', '--format',
'-f',
nargs='?', nargs='?',
choices=['json', 'yaml'], choices=['json', 'yaml'],
help='Specify the format of the input file. Possible values are `json` ' help='Specify the format of the input file. Possible values are `json` '
'and `yaml`. If this option is given, the ' 'and `yaml`. If this option is given, the '
'suffix of the configuration file is ignored.' 'suffix of the configuration file is ignored.',
) )
parser.add_argument( parser.add_argument(
'--send-to', '--send-to',
help='The base URL of the server API. If this option is provided, the ' help='The base URL of the server API. If this option is provided, the '
'configuration will be sent to the server API, otherwise it will just ' 'configuration will be sent to the server API, otherwise it will just '
'be written to stdout.', 'be written to stdout.',
) )
parser.add_argument( parser.add_argument(
'--old-format', '--old-format',
action='store_true', action='store_true',
help='If provided, assume that the configuration is in version 1 format ' help='If provided, assume that the configuration is in version 1 format '
'and convert it to the new format internally (in version 1: tokens ' 'and convert it to the new format internally (in version 1: tokens '
'had no `hashed`-attribute and no `representation`-attribute, the token ' 'had no `hashed`-attribute and no `representation`-attribute, the token '
'representation was the key of the token configuration, ' 'representation was the key of the token configuration, '
'collections had no `schema`-attribute, and `sqlite`-backends had ' 'collections had no `schema`-attribute, and `sqlite`-backends had '
'a `schema`-attribute).', 'a `schema`-attribute).',
) )
parser.add_argument( parser.add_argument(
'--store', '--store',
default=None, default=None,
help='If --old-format is provided, this option can be used to specify a ' help='If --old-format is provided, this option can be used to specify a '
'store directory. The store directory will be used to load `RecordDir` ' 'store directory. The store directory will be used to load `RecordDir` '
'configurations, if a collection defines are `RecordDir`-backend. ' 'configurations, if a collection defines are `RecordDir`-backend. '
'(This option has no effect if no collection in the old configuration ' '(This option has no effect if no collection in the old configuration '
'uses a `RecordDir`-backend.)', 'uses a `RecordDir`-backend.)',
) )
@ -85,16 +85,17 @@ def main():
if arguments.old_format: if arguments.old_format:
configuration = convert_config_1_to_config_2(configuration, arguments.store) configuration = convert_config_1_to_config_2(configuration, arguments.store)
else: elif arguments.store:
if arguments.store: print(
print( 'Warning: ignoring `--store` option because `--old-format` '
'Warning: ignoring `--store` option because `--old-format` ' 'is not provided.',
'is not provided.', file=sys.stderr,
file=sys.stderr, flush=True,
flush=True, )
)
assert configuration['type'] == 'collections', '`type: collections` missing in config-file' assert configuration['type'] == 'collections', (
'`type: collections` missing in config-file'
)
assert configuration['version'] == 2, '`version: 2` missing in config-file' assert configuration['version'] == 2, '`version: 2` missing in config-file'
if arguments.send_to: if arguments.send_to:
@ -110,9 +111,7 @@ def main():
try: try:
establish_configuration( establish_configuration(
configuration, configuration,
arguments.send_to[:-1] arguments.send_to.removesuffix('/'),
if arguments.send_to.endswith('/')
else arguments.send_to,
admin_token, admin_token,
) )
return 0 return 0
@ -135,10 +134,9 @@ def main():
def convert_config_1_to_config_2( def convert_config_1_to_config_2(
old_configuration: dict, old_configuration: dict,
store_path: str | Path, store_path: str | Path,
) -> dict: ) -> dict:
old_version = old_configuration.get('version') old_version = old_configuration.get('version')
if old_version != 1: if old_version != 1:
msg = f'`Unknown old configuration format: {old_version}' msg = f'`Unknown old configuration format: {old_version}'
@ -154,9 +152,11 @@ def convert_config_1_to_config_2(
f'token_{next(counter)}': { f'token_{next(counter)}': {
**old_token_config.copy(), **old_token_config.copy(),
'representation': token_representation, 'representation': token_representation,
'hashed': False 'hashed': False,
} }
for token_representation, old_token_config in old_configuration['tokens'].items() for token_representation, old_token_config in old_configuration[
'tokens'
].items()
} }
old_to_new_token_mapping = { old_to_new_token_mapping = {
@ -165,7 +165,7 @@ def convert_config_1_to_config_2(
} }
store_path = Path(store_path) if store_path else None store_path = Path(store_path) if store_path else None
for collection_name, collection_config in old_configuration['collections'].items(): for collection_config in old_configuration['collections'].values():
backend = collection_config.get('backend') backend = collection_config.get('backend')
if backend and backend['type'].startswith('sqlite'): if backend and backend['type'].startswith('sqlite'):
collection_config['schema'] = backend['schema'] collection_config['schema'] = backend['schema']
@ -174,29 +174,32 @@ def convert_config_1_to_config_2(
if store_path is None: if store_path is None:
msg = '--store <path> has to be provided to convert collection with record_dir-backends' msg = '--store <path> has to be provided to convert collection with record_dir-backends'
raise ValueError(msg) raise ValueError(msg)
record_dir_config = get_record_dir_config(store_path / collection_config['curated']) record_dir_config = get_record_dir_config(
store_path / collection_config['curated']
)
collection_config['schema'] = record_dir_config.schema_location collection_config['schema'] = record_dir_config.schema_location
backend = { backend = {
'type': 'record_dir+stl' if not backend else backend['type'], 'type': 'record_dir+stl' if not backend else backend['type'],
'mapping_method': record_dir_config.idfx.value 'mapping_method': record_dir_config.idfx.value,
} }
collection_config['backend'] = backend collection_config['backend'] = backend
collection_config['default_token'] = old_to_new_token_mapping[collection_config['default_token']] collection_config['default_token'] = old_to_new_token_mapping[
collection_config['default_token']
]
new_configuration = { return {
'type': 'collections', 'type': 'collections',
'version': 2, 'version': 2,
'tokens': new_tokens_dict, 'tokens': new_tokens_dict,
'collections': old_configuration['collections'], 'collections': old_configuration['collections'],
'admin_tokens': {}, 'admin_tokens': {},
} }
return new_configuration
def establish_configuration( def establish_configuration(
configuration: dict, configuration: dict,
api_url: str, api_url: str,
admin_token: str, admin_token: str,
): ):
create_collections(configuration, api_url, admin_token) create_collections(configuration, api_url, admin_token)
create_tokens(configuration, api_url, admin_token) create_tokens(configuration, api_url, admin_token)
@ -204,9 +207,9 @@ def establish_configuration(
def create_tokens( def create_tokens(
configuration: dict, configuration: dict,
api_url: str, api_url: str,
admin_token: str, admin_token: str,
): ):
for token_name, token_config in configuration['tokens'].items(): for token_name, token_config in configuration['tokens'].items():
_post_data( _post_data(
@ -222,9 +225,9 @@ def create_tokens(
def create_collections( def create_collections(
configuration: dict, configuration: dict,
api_url: str, api_url: str,
admin_token: str, admin_token: str,
): ):
for collection_name, collection_config in configuration['collections'].items(): for collection_name, collection_config in configuration['collections'].items():
_post_data( _post_data(
@ -240,9 +243,9 @@ def create_collections(
def create_admin_tokens( def create_admin_tokens(
configuration: dict, configuration: dict,
api_url: str, api_url: str,
admin_token: str, admin_token: str,
): ):
for admin_token_name, admin_token_config in configuration['admin_tokens'].items(): for admin_token_name, admin_token_config in configuration['admin_tokens'].items():
_post_data( _post_data(
@ -258,13 +261,17 @@ def create_admin_tokens(
def _post_data( def _post_data(
url: str, url: str,
data: dict, data: dict,
token: str, token: str,
content_class: str, content_class: str,
content_name: str, content_name: str,
): ):
result = requests.put(url, headers={'x-dumpthings-token': token}, json=data,) result = requests.put(
url,
headers={'x-dumpthings-token': token},
json=data,
)
if result.status_code >= 300: if result.status_code >= 300:
msg = f'Error uploading {content_class}: {content_name}: {result.text}' msg = f'Error uploading {content_class}: {content_name}: {result.text}'
raise RuntimeError(msg) raise RuntimeError(msg)

15
dump_things_service/converter.py
View file

@ -6,10 +6,8 @@ from json import loads as json_loads
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Callable,
) )
from linkml_runtime import SchemaView
from linkml.utils.datautils import ( from linkml.utils.datautils import (
get_dumper, get_dumper,
get_loader, get_loader,
@ -29,10 +27,11 @@ from dump_things_service.model import (
) )
from dump_things_service.utils import cleaned_json from dump_things_service.utils import cleaned_json
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Callable
from types import ModuleType from types import ModuleType
from linkml_runtime import SchemaView
from pydantic import BaseModel from pydantic import BaseModel
from dump_things_service.backends import RecordInfo from dump_things_service.backends import RecordInfo
@ -47,10 +46,7 @@ class TypeValidator:
self.type_name = type_name self.type_name = type_name
self.matcher = None if pattern is None else re.compile(pattern) self.matcher = None if pattern is None else re.compile(pattern)
def validate( def validate(self, value: str) -> str:
self,
value: str
) -> str:
if self.matcher: if self.matcher:
match = self.matcher.match(value) match = self.matcher.match(value)
if not match: if not match:
@ -238,10 +234,7 @@ def _convert_format(
) )
except Exception as e: # BLE001 except Exception as e: # BLE001
if load_only: if load_only:
msg = ( msg = f'Validation error for instance of {target_class}: {e}, data:\n{data}'
f'Validation error for instance of {target_class}: {e}, '
f'data:\n{data}'
)
else: else:
msg = ( msg = (
f'Conversion {input_format} -> {output_format}. Error: {e}, ' f'Conversion {input_format} -> {output_format}. Error: {e}, '

55
dump_things_service/curated.py
View file

@ -1,13 +1,11 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
from itertools import count from typing import TYPE_CHECKING, Annotated
from typing import TYPE_CHECKING
from fastapi import ( from fastapi import (
APIRouter, APIRouter,
Depends, Depends,
FastAPI,
HTTPException, HTTPException,
) )
from fastapi_pagination import ( from fastapi_pagination import (
@ -19,14 +17,13 @@ from fastapi_pagination import (
from dump_things_service import ( from dump_things_service import (
HTTP_401_UNAUTHORIZED, HTTP_401_UNAUTHORIZED,
HTTP_404_NOT_FOUND, HTTP_404_NOT_FOUND,
HTTP_422_UNPROCESSABLE_CONTENT, abstract_config, HTTP_422_UNPROCESSABLE_CONTENT,
) )
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
check_collection, check_collection,
read_config, read_config,
) )
from dump_things_service.api_key import api_key_header_scheme from dump_things_service.api_key import api_key_header_scheme
from dump_things_service.auth import AuthenticationInfo
from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer
from dump_things_service.exceptions import CurieResolutionError from dump_things_service.exceptions import CurieResolutionError
from dump_things_service.instance_state import get_instance_state from dump_things_service.instance_state import get_instance_state
@ -41,6 +38,7 @@ from dump_things_service.utils import (
if TYPE_CHECKING: if TYPE_CHECKING:
from pydantic import BaseModel from pydantic import BaseModel
from dump_things_service.auth import AuthenticationInfo
from dump_things_service.backends import StorageBackend from dump_things_service.backends import StorageBackend
from dump_things_service.lazy_list import LazyList from dump_things_service.lazy_list import LazyList
from dump_things_service.store.model_store import _ModelStore from dump_things_service.store.model_store import _ModelStore
@ -76,7 +74,7 @@ add_pagination(router)
@router.get( @router.get(
'/{collection}/curated/records/{class_name}', '/{collection}/curated/records/{class_name}',
tags=['Curated area: read records'], tags=['Curated area: read records'],
name='Read all records of the given class from the curated area' name='Read all records of the given class from the curated area',
) )
async def read_curated_records_of_type( async def read_curated_records_of_type(
collection: str, collection: str,
@ -104,7 +102,7 @@ async def read_curated_records_of_type(
@router.get( @router.get(
'/{collection}/curated/records/p/{class_name}', '/{collection}/curated/records/p/{class_name}',
tags=['Curated area: read records'], tags=['Curated area: read records'],
name='Read all records of the given class from the curated area with pagination' name='Read all records of the given class from the curated area with pagination',
) )
async def read_curated_records_of_type_paginated( async def read_curated_records_of_type_paginated(
collection: str, collection: str,
@ -112,7 +110,6 @@ async def read_curated_records_of_type_paginated(
matching: str | None = None, matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]: ) -> Page[dict]:
instance_state = get_instance_state() instance_state = get_instance_state()
if class_name not in instance_state.collections[collection].active_classes: if class_name not in instance_state.collections[collection].active_classes:
raise HTTPException( raise HTTPException(
@ -133,7 +130,7 @@ async def read_curated_records_of_type_paginated(
@router.get( @router.get(
'/{collection}/curated/records/', '/{collection}/curated/records/',
tags=['Curated area: read records'], tags=['Curated area: read records'],
name='Read all records from the curated area' name='Read all records from the curated area',
) )
async def read_curated_all_records( async def read_curated_all_records(
collection: str, collection: str,
@ -153,7 +150,7 @@ async def read_curated_all_records(
@router.get( @router.get(
'/{collection}/curated/records/p/', '/{collection}/curated/records/p/',
tags=['Curated area: read records'], tags=['Curated area: read records'],
name='Read all records from the curated area with pagination' name='Read all records from the curated area with pagination',
) )
async def read_curated_all_records_paginated( async def read_curated_all_records_paginated(
collection: str, collection: str,
@ -174,12 +171,12 @@ async def read_curated_all_records_paginated(
@router.get( @router.get(
'/{collection}/curated/record', '/{collection}/curated/record',
tags=['Curated area: read records'], tags=['Curated area: read records'],
name='Read the record with the given pid from the curated area' name='Read the record with the given pid from the curated area',
) )
async def read_curated_record_with_pid( async def read_curated_record_with_pid(
collection: str, collection: str,
pid: str, pid: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
return await _read_curated_records( return await _read_curated_records(
collection=collection, collection=collection,
@ -192,12 +189,12 @@ async def read_curated_record_with_pid(
@router.delete( @router.delete(
'/{collection}/curated/record', '/{collection}/curated/record',
tags=['Curated area: delete records'], tags=['Curated area: delete records'],
name='Delete the record with the given pid from the curated area of the given collection' name='Delete the record with the given pid from the curated area of the given collection',
) )
async def delete_curated_record_with_pid( async def delete_curated_record_with_pid(
collection: str, collection: str,
pid: str, pid: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
return await _delete_curated_record( return await _delete_curated_record(
collection=collection, collection=collection,
@ -214,7 +211,6 @@ async def _read_curated_records(
api_key: str | None = None, api_key: str | None = None,
upper_bound: int | None = 1000, upper_bound: int | None = 1000,
) -> LazyList | dict | None: ) -> LazyList | dict | None:
model_store, backend, _ = _get_store_and_backend(collection, api_key) model_store, backend, _ = _get_store_and_backend(collection, api_key)
if pid: if pid:
@ -232,9 +228,7 @@ async def _read_curated_records(
len(result_list), len(result_list),
upper_bound, upper_bound,
collection, collection,
f'/curated/records/p/{class_name}' f'/curated/records/p/{class_name}' if class_name else '/curated/records/p/',
if class_name
else '/curated/records/p/',
) )
return ModifierList( return ModifierList(
@ -244,9 +238,9 @@ async def _read_curated_records(
async def _delete_curated_record( async def _delete_curated_record(
collection: str, collection: str,
pid: str | None, pid: str | None,
api_key: str | None = None, api_key: str | None = None,
) -> bool: ) -> bool:
with wrap_http_exception(Exception): with wrap_http_exception(Exception):
model_store, backend, _ = _get_store_and_backend(collection, api_key) model_store, backend, _ = _get_store_and_backend(collection, api_key)
@ -255,7 +249,7 @@ async def _delete_curated_record(
raise HTTPException( raise HTTPException(
status_code=HTTP_404_NOT_FOUND, status_code=HTTP_404_NOT_FOUND,
detail=f"Could not remove record with PID '{pid}' from curated area " detail=f"Could not remove record with PID '{pid}' from curated area "
f"of collection '{collection}'.", f"of collection '{collection}'.",
) )
return True return True
@ -264,7 +258,6 @@ def _get_store_and_backend(
collection: str, collection: str,
plain_token: str | None, plain_token: str | None,
) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]: ) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]:
# A token is required # A token is required
if plain_token is None: if plain_token is None:
raise HTTPException( raise HTTPException(
@ -296,14 +289,18 @@ def _get_store_and_backend(
def store_curated_record( def store_curated_record(
collection: str, collection: str,
data: BaseModel, data: BaseModel,
class_name: str, class_name: str,
author_id: str | None = None, author_id: str | None = None,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
): ):
instance_state = get_instance_state() instance_state = get_instance_state()
with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): with wrap_http_exception(
ValueError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Validation error',
):
instance_state.validators[collection].validate(data) instance_state.validators[collection].validate(data)
pid = data.pid pid = data.pid

107
dump_things_service/incoming.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, Annotated
from fastapi import ( from fastapi import (
APIRouter, APIRouter,
@ -22,8 +22,8 @@ from dump_things_service import (
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
check_collection, check_collection,
check_label, check_label,
get_config_labels,
get_config, get_config,
get_config_labels,
) )
from dump_things_service.api_key import api_key_header_scheme from dump_things_service.api_key import api_key_header_scheme
from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer
@ -55,25 +55,27 @@ add_pagination(router)
@router.get( @router.get(
'/{collection}/incoming/', '/{collection}/incoming/',
tags=['Incoming area: read labels'], tags=['Incoming area: read labels'],
name='Get all incoming labels for the given collection' name='Get all incoming labels for the given collection',
) )
async def incoming_read_labels( async def incoming_read_labels(
collection: str, collection: str,
api_key: str | None = Depends(api_key_header_scheme), api_key: Annotated[str | None, Depends(api_key_header_scheme)],
) -> list[str]: ) -> list[str]:
# Authorize api_key # Authorize api_key
await authorize_zones(collection, api_key) await authorize_zones(collection, api_key)
instance_state = get_instance_state() instance_state = get_instance_state()
configured_labels = get_config_labels(get_config(), collection) configured_labels = get_config_labels(get_config(), collection)
on_disk_labels = get_on_disk_labels(instance_state.store_path, get_config(), collection) on_disk_labels = get_on_disk_labels(
instance_state.store_path, get_config(), collection
)
return list(configured_labels.union(on_disk_labels)) return list(configured_labels.union(on_disk_labels))
@router.get( @router.get(
'/{collection}/incoming/{label}/records/{class_name}', '/{collection}/incoming/{label}/records/{class_name}',
tags=['Incoming area: read records'], tags=['Incoming area: read records'],
name='Read all records of the given class from the given incoming area' name='Read all records of the given class from the given incoming area',
) )
async def incoming_read_records_of_type( async def incoming_read_records_of_type(
collection: str, collection: str,
@ -103,7 +105,7 @@ async def incoming_read_records_of_type(
@router.get( @router.get(
'/{collection}/incoming/{label}/records/p/{class_name}', '/{collection}/incoming/{label}/records/p/{class_name}',
tags=['Incoming area: read records'], tags=['Incoming area: read records'],
name='Read all records of the given class from the given incoming area with pagination' name='Read all records of the given class from the given incoming area with pagination',
) )
async def incoming_read_records_of_type_paginated( async def incoming_read_records_of_type_paginated(
collection: str, collection: str,
@ -112,7 +114,6 @@ async def incoming_read_records_of_type_paginated(
matching: str | None = None, matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]: ) -> Page[dict]:
instance_state = get_instance_state() instance_state = get_instance_state()
if class_name not in instance_state.collections[collection].active_classes: if class_name not in instance_state.collections[collection].active_classes:
raise HTTPException( raise HTTPException(
@ -134,7 +135,7 @@ async def incoming_read_records_of_type_paginated(
@router.get( @router.get(
'/{collection}/incoming/{label}/records/', '/{collection}/incoming/{label}/records/',
tags=['Incoming area: read records'], tags=['Incoming area: read records'],
name='Read all records from the given incoming area' name='Read all records from the given incoming area',
) )
async def incoming_read_all_records( async def incoming_read_all_records(
collection: str, collection: str,
@ -156,13 +157,13 @@ async def incoming_read_all_records(
@router.get( @router.get(
'/{collection}/incoming/{label}/records/p/', '/{collection}/incoming/{label}/records/p/',
tags=['Incoming area: read records'], tags=['Incoming area: read records'],
name='Read all records from the given incoming area with pagination' name='Read all records from the given incoming area with pagination',
) )
async def incoming_read_all_records_paginated( async def incoming_read_all_records_paginated(
collection: str, collection: str,
label: str, label: str,
matching: str | None = None, matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]: ) -> Page[dict]:
record_list = await _incoming_read_records( record_list = await _incoming_read_records(
collection=collection, collection=collection,
@ -179,13 +180,13 @@ async def incoming_read_all_records_paginated(
@router.get( @router.get(
'/{collection}/incoming/{label}/record', '/{collection}/incoming/{label}/record',
tags=['Incoming area: read records'], tags=['Incoming area: read records'],
name='Read the record with the given PID from the given incoming area' name='Read the record with the given PID from the given incoming area',
) )
async def incoming_read_record_with_pid( async def incoming_read_record_with_pid(
collection: str, collection: str,
label: str, label: str,
pid: str, pid: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
return await _incoming_read_records( return await _incoming_read_records(
collection=collection, collection=collection,
@ -199,13 +200,13 @@ async def incoming_read_record_with_pid(
@router.delete( @router.delete(
'/{collection}/incoming/{label}/record', '/{collection}/incoming/{label}/record',
tags=['Incoming area: delete records'], tags=['Incoming area: delete records'],
name='Delete the record with the given PID from the given incoming area' name='Delete the record with the given PID from the given incoming area',
) )
async def incoming_delete_record_with_pid( async def incoming_delete_record_with_pid(
collection: str, collection: str,
label: str, label: str,
pid: str, pid: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
return await _incoming_delete_record( return await _incoming_delete_record(
collection=collection, collection=collection,
@ -216,15 +217,14 @@ async def incoming_delete_record_with_pid(
async def _incoming_read_records( async def _incoming_read_records(
collection: str, collection: str,
label: str, label: str,
class_name: str | None, class_name: str | None,
pid: str | None, pid: str | None,
matching: str | None = None, matching: str | None = None,
api_key: str | None = None, api_key: str | None = None,
upper_bound: int = 1000, upper_bound: int = 1000,
) -> LazyList | dict | None: ) -> LazyList | dict | None:
model_store, backend = await _get_store_and_backend(collection, label, api_key) model_store, backend = await _get_store_and_backend(collection, label, api_key)
if pid: if pid:
@ -244,7 +244,7 @@ async def _incoming_read_records(
collection, collection,
f'/incoming/{label}/records/p/{class_name}' f'/incoming/{label}/records/p/{class_name}'
if class_name if class_name
else f'/incoming/{label}/records/p/' else f'/incoming/{label}/records/p/',
) )
return ModifierList( return ModifierList(
@ -266,7 +266,7 @@ async def _incoming_delete_record(
raise HTTPException( raise HTTPException(
status_code=HTTP_404_NOT_FOUND, status_code=HTTP_404_NOT_FOUND,
detail=f"Could not remove record with PID '{pid}' from incoming " detail=f"Could not remove record with PID '{pid}' from incoming "
f"area '{label}' of collection '{collection}'.", f"area '{label}' of collection '{collection}'.",
) )
return True return True
@ -276,7 +276,6 @@ async def _get_store_and_backend(
label: str, label: str,
plain_token: str | None, plain_token: str | None,
) -> tuple[_ModelStore, StorageBackend]: ) -> tuple[_ModelStore, StorageBackend]:
# Authorize api_key # Authorize api_key
await authorize_zones(collection, plain_token) await authorize_zones(collection, plain_token)
@ -301,33 +300,6 @@ async def _get_store_and_backend(
store_dir=store_dir, store_dir=store_dir,
) )
xxx = """
# For consistency, associate the store with all matching tokens from the
# configuration file. That means with all tokens that have the same
# input
matching_tokens = [
token_name
for token_name, token_info in abstract_config.tokens.items()
if (collection, label) in [
(collection_name, token_collection_info.incoming_label)
for collection_name, token_collection_info in token_info.items()
]
]
for matching_token in matching_tokens:
# Associate the store with all matching tokens in the configuration.
# Note: there are stores that are not associated with a token in
# the abstract configuration. These are stores that belong to a token
# that is authenticated with an external authentication source.
token_info = instance_state.tokens[collection][matching_token]
instance_state.token_stores[collection][matching_token] = (
model_store,
matching_token,
token_info['permissions'],
token_info['user_id'],
)
"""
backend = model_store.backend backend = model_store.backend
if isinstance(backend, _SchemaTypeLayer): if isinstance(backend, _SchemaTypeLayer):
return model_store, backend.backend return model_store, backend.backend
@ -361,15 +333,18 @@ async def authorize_zones(
async def store_incoming_record( async def store_incoming_record(
collection: str, collection: str,
label: str, label: str,
data: BaseModel, data: BaseModel,
class_name: str, class_name: str,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
): ):
instance_state = get_instance_state() instance_state = get_instance_state()
with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): with wrap_http_exception(
ValueError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Validation error',
):
instance_state.validators[collection].validate(data) instance_state.validators[collection].validate(data)
pid = data.pid pid = data.pid

32
dump_things_service/instance_state.py
View file

@ -3,25 +3,20 @@ from __future__ import annotations
import dataclasses import dataclasses
import logging import logging
from functools import cache from functools import cache
from pathlib import Path
from types import ModuleType
from typing import ( from typing import (
TYPE_CHECKING,
Any, Any,
Callable,
) )
import yaml import yaml
from fastapi import FastAPI
from linkml_runtime import SchemaView
from pydantic import ValidationError from pydantic import ValidationError
from yaml.scanner import ScannerError from yaml.scanner import ScannerError
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
RecordDirConfigFileContent,
MappingMethod, MappingMethod,
RecordDirConfigFileContent,
mapping_functions, mapping_functions,
) )
from dump_things_service.converter import get_conversion_objects from dump_things_service.converter import get_conversion_objects
from dump_things_service.exceptions import ConfigError from dump_things_service.exceptions import ConfigError
from dump_things_service.model import ( from dump_things_service.model import (
@ -30,6 +25,13 @@ from dump_things_service.model import (
get_schema_view, get_schema_view,
) )
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from types import ModuleType
from fastapi import FastAPI
from linkml_runtime import SchemaView
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
@ -86,7 +88,9 @@ class InstanceState:
maintenance_mode: set = dataclasses.field(default_factory=set) maintenance_mode: set = dataclasses.field(default_factory=set)
# Created based on abstract configuration # Created based on abstract configuration
collections: dict[str, InstanceStateCollectionInfo] = dataclasses.field(default_factory=dict) collections: dict[str, InstanceStateCollectionInfo] = dataclasses.field(
default_factory=dict
)
tokens: dict = dataclasses.field(default_factory=dict) tokens: dict = dataclasses.field(default_factory=dict)
auth_sources: dict[str, list] = dataclasses.field(default_factory=dict) auth_sources: dict[str, list] = dataclasses.field(default_factory=dict)
audit_backends: dict[str, list] = dataclasses.field(default_factory=dict) audit_backends: dict[str, list] = dataclasses.field(default_factory=dict)
@ -97,13 +101,13 @@ class InstanceState:
order_by: list[str] = dataclasses.field(default_factory=list) order_by: list[str] = dataclasses.field(default_factory=list)
g_instance_state:InstanceState | None = None g_instance_state: InstanceState | None = None
def create_instance_state( def create_instance_state(
store_path: Path, store_path: Path,
bootstrap_token: str, bootstrap_token: str,
fastapi_app: FastAPI, fastapi_app: FastAPI,
) -> InstanceState: ) -> InstanceState:
global g_instance_state global g_instance_state
@ -128,8 +132,8 @@ def get_instance_state() -> InstanceState:
def get_record_dir_config( def get_record_dir_config(
path: Path, path: Path,
file_name: str = record_dir_config_file_name, file_name: str = record_dir_config_file_name,
) -> RecordDirConfigFileContent: ) -> RecordDirConfigFileContent:
config_path = path / file_name config_path = path / file_name
if not config_path.exists(): if not config_path.exists():

5
dump_things_service/lazy_list.py
View file

@ -27,10 +27,9 @@ from abc import (
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Iterable from collections.abc import Callable, Iterable
from typing import ( from typing import (
Any, Any,
Callable,
) )
@ -177,7 +176,7 @@ class PriorityList(LazyList):
""" """
def __init__( def __init__(
self, self,
): ):
super().__init__() super().__init__()
self.seen = set() self.seen = set()

179
dump_things_service/main.py
View file

@ -5,13 +5,14 @@ import logging
import os import os
import sys import sys
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, Annotated
from dump_things_service.abstract_config import store_config from dump_things_service.abstract_config import store_config
from dump_things_service.commands.upload_config import convert_config_1_to_config_2 from dump_things_service.commands.upload_config import convert_config_1_to_config_2
from dump_things_service.manifest import manifest_configuration from dump_things_service.manifest import manifest_configuration
# Perform the patching before importing any third-party libraries # Perform the patching before importing any third-party libraries
from dump_things_service.patches import enabled # noqa F401 -- used by generated code from dump_things_service.patches import enabled # noqa: F401 -- used by generated code
import yaml import yaml
import uvicorn import uvicorn
@ -57,8 +58,7 @@ from dump_things_service.converter import (
from dump_things_service.curated import router as curated_router from dump_things_service.curated import router as curated_router
from dump_things_service.exceptions import CurieResolutionError from dump_things_service.exceptions import CurieResolutionError
from dump_things_service.incoming import router as incoming_router from dump_things_service.incoming import router as incoming_router
from dump_things_service.instance_state import create_instance_state, \ from dump_things_service.instance_state import create_instance_state, InstanceState
InstanceState
from dump_things_service.lazy_list import ( from dump_things_service.lazy_list import (
PriorityList, PriorityList,
ModifierList, ModifierList,
@ -97,7 +97,7 @@ class ServerCollectionCountedResponse(ServerCollectionResponse):
class ServerResponse(BaseModel): class ServerResponse(BaseModel):
version: str version: str
collections: list[ServerCollectionResponse|ServerCollectionCountedResponse] collections: list[ServerCollectionResponse | ServerCollectionCountedResponse]
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
@ -106,7 +106,7 @@ logger = logging.getLogger('dump_things_service')
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--host', default='0.0.0.0') # noqa S104 parser.add_argument('--host', default='0.0.0.0') # noqa: S104
parser.add_argument('--port', default=8000, type=int) parser.add_argument('--port', default=8000, type=int)
parser.add_argument('--origins', action='append', default=[]) parser.add_argument('--origins', action='append', default=[])
parser.add_argument( parser.add_argument(
@ -114,19 +114,19 @@ parser.add_argument(
type=str, type=str,
default='', default='',
help='The sha256 hash of an initial admin token that will allow to add or ' help='The sha256 hash of an initial admin token that will allow to add or '
'remove tokens, collections, and additional admin tokens (64 ' 'remove tokens, collections, and additional admin tokens (64 '
'characters hex-digit). NOTE: an admin token in plaintext is read ' 'characters hex-digit). NOTE: an admin token in plaintext is read '
'from the environment variable `DTS_ADMIN_TOKEN` if it is set, and ' 'from the environment variable `DTS_ADMIN_TOKEN` if it is set, and '
'if this option is not provided.', 'if this option is not provided.',
) )
parser.add_argument( parser.add_argument(
'-c', '-c',
'--config', '--config',
metavar='CONFIG_FILE', metavar='CONFIG_FILE',
help="Read the configuration from 'CONFIG_FILE' if no persisted " help="Read the configuration from 'CONFIG_FILE' if no persisted "
"configuration is found in the data store root directory, and " 'configuration is found in the data store root directory, and '
"initialize the persistent configuration and the service state with " 'initialize the persistent configuration and the service state with '
"the values in 'CONFIG_FILE'.", "the values in 'CONFIG_FILE'.",
) )
parser.add_argument( parser.add_argument(
'--root-path', '--root-path',
@ -141,10 +141,10 @@ parser.add_argument(
parser.add_argument( parser.add_argument(
'--ignore-default-config-file', '--ignore-default-config-file',
action='store_true', action='store_true',
help="If the persisted configuration is empty, do not try to initialize it " help='If the persisted configuration is empty, do not try to initialize it '
"from an existing default-config file, i.e., do not read the file " 'from an existing default-config file, i.e., do not read the file '
"`<store>/.dumpthings.yaml`. That means the configuration be empty " '`<store>/.dumpthings.yaml`. That means the configuration be empty '
"collections and tokens are added via the API.", 'collections and tokens are added via the API.',
) )
parser.add_argument( parser.add_argument(
'store', 'store',
@ -182,15 +182,14 @@ if not arguments.admin_token_hash:
arguments.admin_token_hash = hash_token_representation( arguments.admin_token_hash = hash_token_representation(
os.environ.get('DTS_ADMIN_TOKEN', ''), os.environ.get('DTS_ADMIN_TOKEN', ''),
) )
else: # Validate the hash token format
# Validate the hash token format elif not hash_matcher.match(arguments.admin_token_hash):
if not hash_matcher.match(arguments.admin_token_hash): print(
print( 'Hashed admin token is not a 64-digits hex-number',
'Hashed admin token is not a 64-digits hex-number', file=sys.stderr,
file=sys.stderr, flush=True,
flush=True, )
) sys.exit(1)
sys.exit(1)
# Set the log level # Set the log level
@ -247,8 +246,8 @@ g_configuration = read_config(store_path)
def initialize_from_config_file( def initialize_from_config_file(
instance_state: InstanceState, instance_state: InstanceState,
config_file: str | Path, config_file: str | Path,
) -> Configuration: ) -> Configuration:
with open(config_file) as f: with open(config_file) as f:
config_dict = yaml.safe_load(f) config_dict = yaml.safe_load(f)
@ -274,20 +273,20 @@ def initialize_from_config_file(
# location, i.e., from `<store>/.dumpthings.yaml`, or from the configuration # location, i.e., from `<store>/.dumpthings.yaml`, or from the configuration
# option, unless `--dont-use-old-config` is specified. # option, unless `--dont-use-old-config` is specified.
if not ( if not (
g_configuration.admin_tokens g_configuration.admin_tokens
or g_configuration.collections or g_configuration.collections
or g_configuration.tokens or g_configuration.tokens
): ):
if arguments.config: if arguments.config:
config_file = arguments.config config_file = arguments.config
elif arguments.ignore_default_config_file:
config_file = None
else: else:
if arguments.ignore_default_config_file: from dump_things_service import config_file_name
config_file = g_instance_state.store_path / config_file_name
if not config_file.exists():
config_file = None config_file = None
else:
from dump_things_service import config_file_name
config_file = g_instance_state.store_path / config_file_name
if not config_file.exists():
config_file = None
if config_file: if config_file:
logger.info( logger.info(
@ -307,18 +306,17 @@ if not (
# If there are no structures in the configuration, check for a bootstrap token. # If there are no structures in the configuration, check for a bootstrap token.
if not ( if not (
g_configuration.admin_tokens g_configuration.admin_tokens
or g_configuration.collections or g_configuration.collections
or g_configuration.tokens or g_configuration.tokens
): ) and not g_instance_state.bootstrap_token:
if not g_instance_state.bootstrap_token: print(
print( 'The server has an empty configuration and requires a bootstrap '
'The server has an empty configuration and requires a bootstrap ' 'token (use `--admin-token-hash` to provide one)',
'token (use `--admin-token-hash` to provide one).', file=sys.stderr,
file=sys.stderr, flush=True,
flush=True, )
) sys.exit(2)
sys.exit(2)
manifest_configuration( manifest_configuration(
@ -337,38 +335,36 @@ async def root() -> RedirectResponse:
return RedirectResponse('/docs') return RedirectResponse('/docs')
@app.get( @app.get('/server', tags=['Server management'], name='get server information')
'/server',
tags=['Server management'],
name='get server information'
)
async def server() -> ServerResponse: async def server() -> ServerResponse:
return ServerResponse( return ServerResponse(
version = __version__, version=__version__,
collections = [ collections=[
ServerCollectionResponse( ServerCollectionResponse(
name=collection_name, name=collection_name,
schema=g_configuration.collections[collection_name].schema_location, schema=g_configuration.collections[collection_name].schema_location,
classes=g_instance_state.schema_info[g_configuration.collections[collection_name].schema_location].classes, classes=g_instance_state.schema_info[
g_configuration.collections[collection_name].schema_location
].classes,
) )
for collection_name in g_configuration.collections for collection_name in g_configuration.collections
] ],
) )
@app.post( @app.post(
'/maintenance', '/maintenance',
tags=['Server management'], tags=['Server management'],
name='put a collection in maintenance mode' name='put a collection in maintenance mode',
) )
async def maintenance( async def maintenance(
body: MaintenanceRequest, body: MaintenanceRequest,
api_key: str | None = Depends(api_key_header_scheme), api_key: Annotated[str | None, Depends(api_key_header_scheme)],
): ):
if api_key is None: if api_key is None:
raise HTTPException( raise HTTPException(
status_code=HTTP_400_BAD_REQUEST, status_code=HTTP_400_BAD_REQUEST,
detail=f'Token required for this operation', detail='Token required for this operation',
) )
collection = body.collection collection = body.collection
@ -381,20 +377,19 @@ async def maintenance(
permissions = auth_info.token_permission permissions = auth_info.token_permission
if not ( if not (
permissions.curated_write permissions.curated_write
and permissions.curated_read and permissions.curated_read
and permissions.zones_access and permissions.zones_access
): ):
raise HTTPException( raise HTTPException(
status_code=HTTP_400_BAD_REQUEST, status_code=HTTP_400_BAD_REQUEST,
detail=f'Curator permissions required for this operation', detail='Curator permissions required for this operation',
) )
if active: if active:
g_instance_state.maintenance_mode.add(collection) g_instance_state.maintenance_mode.add(collection)
else: else:
g_instance_state.maintenance_mode.remove(collection) g_instance_state.maintenance_mode.remove(collection)
return
@app.get( @app.get(
@ -405,7 +400,7 @@ async def maintenance(
async def read_record_with_pid( async def read_record_with_pid(
collection: str, collection: str,
pid: str, pid: str,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
): ):
check_collection(g_configuration, collection) check_collection(g_configuration, collection)
@ -446,10 +441,10 @@ async def read_record_with_pid(
name='Read all records from the given collection', name='Read all records from the given collection',
) )
async def read_all_records( async def read_all_records(
collection: str, collection: str,
matching: str | None = None, matching: str | None = None,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
): ):
return await _read_all_records( return await _read_all_records(
collection=collection, collection=collection,
@ -469,10 +464,10 @@ async def read_all_records(
name='Read all records from the given collection with pagination', name='Read all records from the given collection with pagination',
) )
async def read_all_records_paginated( async def read_all_records_paginated(
collection: str, collection: str,
matching: str | None = None, matching: str | None = None,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
) -> Page[dict | str]: ) -> Page[dict | str]:
result_list = await _read_all_records( result_list = await _read_all_records(
collection=collection, collection=collection,
@ -493,7 +488,7 @@ async def read_records_of_type(
collection: str, collection: str,
class_name: str, class_name: str,
matching: str | None = None, matching: str | None = None,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
): ):
return await _read_records_of_type( return await _read_records_of_type(
@ -518,7 +513,7 @@ async def read_records_of_type_paginated(
collection: str, collection: str,
class_name: str, class_name: str,
matching: str | None = None, matching: str | None = None,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
) -> Page[dict | str]: ) -> Page[dict | str]:
result_list = await _read_records_of_type( result_list = await _read_records_of_type(
@ -533,13 +528,12 @@ async def read_records_of_type_paginated(
async def _read_all_records( async def _read_all_records(
collection: str, collection: str,
matching: str | None = None, matching: str | None = None,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
bound: int | None = None, bound: int | None = None,
) -> LazyList: ) -> LazyList:
def convert_to_http_exception(e: BaseException): def convert_to_http_exception(e: BaseException):
raise HTTPException( raise HTTPException(
status_code=HTTP_400_BAD_REQUEST, status_code=HTTP_400_BAD_REQUEST,
@ -591,7 +585,7 @@ async def _read_records_of_type(
collection: str, collection: str,
class_name: str, class_name: str,
matching: str | None = None, matching: str | None = None,
format: Format = Format.json, # noqa A002 format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme), api_key: str = Depends(api_key_header_scheme),
bound: int | None = None, bound: int | None = None,
) -> LazyList: ) -> LazyList:
@ -622,7 +616,9 @@ async def _read_records_of_type(
matching=matching, matching=matching,
) )
if bound: if bound:
check_bounds(len(token_store_list), bound, collection, f'/records/p/{class_name}') check_bounds(
len(token_store_list), bound, collection, f'/records/p/{class_name}'
)
result_list.add_list(token_store_list) result_list.add_list(token_store_list)
if final_permissions.curated_read: if final_permissions.curated_read:
@ -634,7 +630,12 @@ async def _read_records_of_type(
matching=matching, matching=matching,
) )
if bound: if bound:
check_bounds(len(curated_store_list), bound, collection, f'/records/p/{class_name}') check_bounds(
len(curated_store_list),
bound,
collection,
f'/records/p/{class_name}',
)
result_list.add_list(curated_store_list) result_list.add_list(curated_store_list)
# Sort the result list. # Sort the result list.
@ -664,7 +665,7 @@ async def _read_records_of_type(
async def delete_record( async def delete_record(
collection: str, collection: str,
pid: str, pid: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
check_collection(g_configuration, collection) check_collection(g_configuration, collection)
final_permissions, token_store = await process_token( final_permissions, token_store = await process_token(
@ -682,8 +683,8 @@ async def delete_record(
raise HTTPException( raise HTTPException(
status_code=HTTP_404_NOT_FOUND, status_code=HTTP_404_NOT_FOUND,
detail=f"Could not remove record with PID '{pid}' from the " detail=f"Could not remove record with PID '{pid}' from the "
"token associated incoming area of collection " 'token associated incoming area of collection '
f"'{collection}'.", f"'{collection}'.",
) )
return True return True

25
dump_things_service/manifest.py
View file

@ -12,7 +12,6 @@ from dump_things_service.collection import (
) )
from dump_things_service.instance_state import InstanceState from dump_things_service.instance_state import InstanceState
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
tag_groups = [ tag_groups = [
@ -58,10 +57,9 @@ openapi_tags_template = [
] ]
def manifest_configuration( def manifest_configuration(
configuration: Configuration, configuration: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
): ):
"""Interpret the configuration and instantiate respective objects """Interpret the configuration and instantiate respective objects
@ -160,23 +158,23 @@ def manifest_configuration(
def create_token( def create_token(
instance_state: InstanceState, instance_state: InstanceState,
token_name: str, token_name: str,
token_configuration: TokenConfig, token_configuration: TokenConfig,
): ):
instance_state.tokens[token_name] = token_configuration instance_state.tokens[token_name] = token_configuration
def delete_token( def delete_token(
instance_state: InstanceState, instance_state: InstanceState,
token_name: str, token_name: str,
): ):
instance_state.tokens.pop(token_name) instance_state.tokens.pop(token_name)
def delete_collection( def delete_collection(
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
): ):
instance_state.collections.pop(collection_name) instance_state.collections.pop(collection_name)
@ -190,7 +188,6 @@ def create_openapi_tags(
instance_state: InstanceState, instance_state: InstanceState,
openapi_tags_template: list[dict | str], openapi_tags_template: list[dict | str],
) -> list[dict]: ) -> list[dict]:
# Collect tag name lists for all tag groups that we have defined. # Collect tag name lists for all tag groups that we have defined.
tag_group_info = { tag_group_info = {
tag_group: sorted( tag_group: sorted(
@ -198,12 +195,12 @@ def create_openapi_tags(
{'name': collection_info.tag_info[tag_group]} {'name': collection_info.tag_info[tag_group]}
for collection_info in instance_state.collections.values() for collection_info in instance_state.collections.values()
], ],
key=lambda x: x['name'] key=lambda x: x['name'],
) )
for tag_group in tag_groups for tag_group in tag_groups
} }
result = openapi_tags_template.copy() result = openapi_tags_template.copy()
for tag_group, tag_list in tag_group_info.items(): for tag_group, tag_list in tag_group_info.items():
index = result.index(tag_group) index = result.index(tag_group)
result[index:index + 1] = tag_list result[index : index + 1] = tag_list
return result return result

20
dump_things_service/model.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
import dataclasses # noqa F401 -- used by generated code import dataclasses # noqa: F401 -- used by generated code
import logging import logging
import sys import sys
from functools import cache from functools import cache
@ -11,9 +11,9 @@ from typing import (
) )
from urllib.parse import urlparse from urllib.parse import urlparse
import annotated_types # noqa F401 -- used by generated code import annotated_types # noqa: F401 -- used by generated code
import pydantic # noqa F401 -- used by generated code import pydantic # noqa: F401 -- used by generated code
import pydantic_core # noqa F401 -- used by generated code import pydantic_core # noqa: F401 -- used by generated code
from linkml.generators import ( from linkml.generators import (
PydanticGenerator, PydanticGenerator,
PythonGenerator, PythonGenerator,
@ -22,7 +22,7 @@ from linkml_runtime import SchemaView
from pydantic._internal._model_construction import ModelMetaclass from pydantic._internal._model_construction import ModelMetaclass
# Ensure linkml is patched # Ensure linkml is patched
import dump_things_service.patches.enabled # noqa F401 -- apply patches import dump_things_service.patches.enabled # noqa: F401 -- apply patches
if TYPE_CHECKING: if TYPE_CHECKING:
from types import ModuleType from types import ModuleType
@ -67,11 +67,11 @@ def get_subclasses(
# TODO: shall we use the following code? # TODO: shall we use the following code?
# The code below would use schema-definitions to determine classes and not # The code below would use schema-definitions to determine classes and not
# go through thw pydantic module generation. # go through the pydantic module generation.
@cache @cache
def get_subclasses_2( def get_subclasses_2(
collection_name: str, collection_name: str,
class_name: str, class_name: str,
) -> list[str]: ) -> list[str]:
from dump_things_service.instance_state import get_instance_state from dump_things_service.instance_state import get_instance_state
@ -88,8 +88,8 @@ def compile_module_with_increasing_recursion_limit(
module = None module = None
module_name = ( module_name = (
urlparse(schema_location).path urlparse(schema_location)
.replace('/', '_') .path.replace('/', '_')
.replace('-', '_') .replace('-', '_')
.replace('.', '_') .replace('.', '_')
) )

89
dump_things_service/store/model_store.py
View file

@ -17,8 +17,8 @@ if TYPE_CHECKING:
from pydantic import BaseModel from pydantic import BaseModel
from dump_things_service.backends import ( from dump_things_service.backends import (
_RecordInfo,
StorageBackend, StorageBackend,
_RecordInfo,
) )
from dump_things_service.lazy_list import LazyList from dump_things_service.lazy_list import LazyList
@ -28,12 +28,7 @@ submitter_namespace = 'http://purl.obolibrary.org/obo/'
class _ModelStore: class _ModelStore:
def __init__( def __init__(self, schema: str, backend: StorageBackend, tags: dict[str, str]):
self,
schema: str,
backend: StorageBackend,
tags: dict[str, str]
):
self.schema = schema self.schema = schema
self.model = get_model_for_schema(self.schema)[0] self.model = get_model_for_schema(self.schema)[0]
self.backend = backend self.backend = backend
@ -43,11 +38,13 @@ class _ModelStore:
return self.backend.get_uri() return self.backend.get_uri()
def store_object( def store_object(
self, self,
obj: BaseModel, obj: BaseModel,
submitter: str | None, submitter: str | None,
) -> Iterable[tuple[str, dict]]: ) -> Iterable[tuple[str, dict]]:
if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in (obj.annotations or dict()): if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in (
obj.annotations or {}
):
return [] return []
# Extract inlined records from the object, store individual records # Extract inlined records from the object, store individual records
@ -64,15 +61,15 @@ class _ModelStore:
] ]
def pid_to_iri( def pid_to_iri(
self, self,
pid: str, pid: str,
): ):
return resolve_curie(self.model, pid) return resolve_curie(self.model, pid)
def _store_flat_object( def _store_flat_object(
self, self,
obj: BaseModel, obj: BaseModel,
submitter: str | None, submitter: str | None,
) -> dict: ) -> dict:
iri = self.pid_to_iri(obj.pid) iri = self.pid_to_iri(obj.pid)
class_name = obj.__class__.__name__ class_name = obj.__class__.__name__
@ -94,9 +91,9 @@ class _ModelStore:
return json_object return json_object
def annotate( def annotate(
self, self,
json_object: dict, json_object: dict,
submitter: str, submitter: str,
) -> None: ) -> None:
"""Add submitter IRI to the record annotations, use CURIE if possible""" """Add submitter IRI to the record annotations, use CURIE if possible"""
json_object['annotations'] = self.homogenize_annotations(json_object) json_object['annotations'] = self.homogenize_annotations(json_object)
@ -113,8 +110,8 @@ class _ModelStore:
} }
def get_curie( def get_curie(
self, self,
curie_or_iri: str, curie_or_iri: str,
) -> str: ) -> str:
if is_curie(curie_or_iri): if is_curie(curie_or_iri):
return curie_or_iri return curie_or_iri
@ -131,8 +128,8 @@ class _ModelStore:
return curie_or_iri return curie_or_iri
def extract_inlined( def extract_inlined(
self, self,
record: BaseModel, record: BaseModel,
) -> list[BaseModel]: ) -> list[BaseModel]:
# The trivial case: no relations # The trivial case: no relations
if not hasattr(record, 'relations') or record.relations is None: if not hasattr(record, 'relations') or record.relations is None:
@ -146,7 +143,8 @@ class _ModelStore:
# Do not extract 'empty'-Thing records with an # Do not extract 'empty'-Thing records with an
# `dlthings:placeholder` annotation. These records are just # `dlthings:placeholder` annotation. These records are just
# placeholders for already extracted records. # placeholders for already extracted records.
if sub_record != self.model.Thing( if sub_record
!= self.model.Thing(
pid=sub_record.pid, pid=sub_record.pid,
annotations={ annotations={
'dlthings:placeholder': sub_record.pid, 'dlthings:placeholder': sub_record.pid,
@ -165,21 +163,21 @@ class _ModelStore:
pid=sub_record_pid, pid=sub_record_pid,
annotations={ annotations={
'dlthings:placeholder': sub_record_pid, 'dlthings:placeholder': sub_record_pid,
} },
) )
for sub_record_pid in record.relations for sub_record_pid in record.relations
} }
return [new_record, *extracted_sub_records] return [new_record, *extracted_sub_records]
def get_object_by_pid( def get_object_by_pid(
self, self,
pid: str, pid: str,
) -> tuple[str, dict] | tuple[None, None]: ) -> tuple[str, dict] | tuple[None, None]:
return self.get_object_by_iri(self.pid_to_iri(pid)) return self.get_object_by_iri(self.pid_to_iri(pid))
def get_object_by_iri( def get_object_by_iri(
self, self,
iri: str, iri: str,
) -> tuple[str, dict] | tuple[None, None]: ) -> tuple[str, dict] | tuple[None, None]:
record_info = self.backend.get_record_by_iri(iri) record_info = self.backend.get_record_by_iri(iri)
if record_info: if record_info:
@ -187,11 +185,11 @@ class _ModelStore:
return None, None return None, None
def get_objects_of_class( def get_objects_of_class(
self, self,
class_name: str, class_name: str,
matching: str | None, matching: str | None,
*, *,
include_subclasses: bool = True, include_subclasses: bool = True,
) -> LazyList[_RecordInfo]: ) -> LazyList[_RecordInfo]:
""" """
Get all objects of a specific class. Get all objects of a specific class.
@ -210,8 +208,8 @@ class _ModelStore:
return self.backend.get_records_of_classes(class_names, matching) return self.backend.get_records_of_classes(class_names, matching)
def get_all_objects( def get_all_objects(
self, self,
matching: str | None = None, matching: str | None = None,
) -> LazyList[_RecordInfo]: ) -> LazyList[_RecordInfo]:
""" """
Get all objects of a specific class. Get all objects of a specific class.
@ -222,8 +220,8 @@ class _ModelStore:
return self.backend.get_all_records(matching) return self.backend.get_all_records(matching)
def delete_object( def delete_object(
self, self,
pid: str, pid: str,
) -> bool: ) -> bool:
return self.backend.remove_record(self.pid_to_iri(pid)) return self.backend.remove_record(self.pid_to_iri(pid))
@ -232,9 +230,9 @@ _existing_model_stores = {}
def ModelStore( # noqa: N802 def ModelStore( # noqa: N802
schema: str, schema: str,
backend: StorageBackend, backend: StorageBackend,
tags: dict[str, str], tags: dict[str, str],
) -> _ModelStore: ) -> _ModelStore:
"""Create a unique model store for the given schema and backend. """Create a unique model store for the given schema and backend.
@ -252,10 +250,9 @@ def ModelStore( # noqa: N802
# We store a pointer to the backend in the value to ensure that the # We store a pointer to the backend in the value to ensure that the
# backend object exists while we use its `id` as a key. # backend object exists while we use its `id` as a key.
_existing_model_stores[id(backend)] = existing_model_store, backend _existing_model_stores[id(backend)] = existing_model_store, backend
else: # Check that the schemas are compatible, if the backend is reused.
# Check that the schemas are compatible, if the backend is reused. elif existing_model_store.schema != schema:
if existing_model_store.schema != schema: msg = 'Backend is already used in a ModelStore with a different schema'
msg = 'Backend is already used in a ModelStore with a different schema' raise ValueError(msg)
raise ValueError(msg)
return existing_model_store return existing_model_store

13
dump_things_service/tests/create_store.py
View file

@ -4,18 +4,19 @@ from typing import TYPE_CHECKING
import yaml import yaml
from dump_things_service.backends.record_dir import RecordDirStore
from dump_things_service.backends.sqlite import (
SQLiteBackend,
record_file_name as sqlite_record_file_name,
)
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
RecordDirBackendConfig,
CollectionConfig, CollectionConfig,
Configuration, Configuration,
MappingMethod, MappingMethod,
RecordDirBackendConfig,
mapping_functions, mapping_functions,
) )
from dump_things_service.backends.sqlite import (
SQLiteBackend,
)
from dump_things_service.backends.sqlite import (
record_file_name as sqlite_record_file_name,
)
from dump_things_service.model import get_model_for_schema from dump_things_service.model import get_model_for_schema
from dump_things_service.resolve_curie import resolve_curie from dump_things_service.resolve_curie import resolve_curie

92
dump_things_service/tests/fixtures.py
View file

@ -11,20 +11,23 @@ import yaml
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
GitAuditBackendConfig, GitAuditBackendConfig,
SQLiteBackendConfig, SQLiteBackendConfig,
TagSpec,
TokenCollectionConfig, TokenCollectionConfig,
TokenModes, hash_token_representation, TagSpec, TokenModes,
hash_token_representation,
) )
from dump_things_service.backends import StorageBackend from dump_things_service.backends import StorageBackend
from dump_things_service.backends.record_dir import RecordDirStore from dump_things_service.backends.record_dir import RecordDirStore
from dump_things_service.backends.sqlite import ( from dump_things_service.backends.sqlite import (
SQLiteBackend, SQLiteBackend,
)
from dump_things_service.backends.sqlite import (
record_file_name as sqlite_db_filename, record_file_name as sqlite_db_filename,
) )
from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.collection_endpoints import CollectionRequest
from dump_things_service.instance_state import get_mapping_function_by_name from dump_things_service.instance_state import get_mapping_function_by_name
from dump_things_service.model import get_model_for_schema from dump_things_service.model import get_model_for_schema
from dump_things_service.resolve_curie import resolve_curie from dump_things_service.resolve_curie import resolve_curie
from dump_things_service.token_endpoints import TokenRequest
from dump_things_service.tests.create_store import ( from dump_things_service.tests.create_store import (
pid, pid,
pid_curated, pid_curated,
@ -33,7 +36,7 @@ from dump_things_service.tests.create_store import (
test_record_curated, test_record_curated,
test_record_trr, test_record_trr,
) )
from dump_things_service.token_endpoints import TokenRequest
# String representation of curated- and incoming-path # String representation of curated- and incoming-path
curated = 'curated' curated = 'curated'
@ -41,7 +44,9 @@ incoming = 'incoming'
# Path to a local simple test schema # Path to a local simple test schema
test_schema_location = str((Path(__file__).parent / 'testschema.yaml').absolute()) test_schema_location = str((Path(__file__).parent / 'testschema.yaml').absolute())
flat_social_schema_location = 'https://concepts.datalad.org/s/flat-social/unreleased.yaml' flat_social_schema_location = (
'https://concepts.datalad.org/s/flat-social/unreleased.yaml'
)
# The test store is created empty and collections are added via the admin # The test store is created empty and collections are added via the admin
@ -64,7 +69,7 @@ g_default_collections[6].submission_tags = TagSpec(
g_default_collections.append( g_default_collections.append(
CollectionRequest( CollectionRequest(
name=f'collection_8', name='collection_8',
default_token='test_default_token', default_token='test_default_token',
curated=PurePosixPath(f'{curated}/collection_8'), curated=PurePosixPath(f'{curated}/collection_8'),
schema=test_schema_location, schema=test_schema_location,
@ -75,38 +80,40 @@ g_default_collections.append(
submission_tags=TagSpec( submission_tags=TagSpec(
submitter_id_tag='no_default_id_tag', submitter_id_tag='no_default_id_tag',
submission_time_tag='no_default_time_tag', submission_time_tag='no_default_time_tag',
) ),
) )
) )
g_default_collections.extend([ g_default_collections.extend(
CollectionRequest( [
name='collection_dlflatsocial-1', CollectionRequest(
schema=flat_social_schema_location, name='collection_dlflatsocial-1',
default_token='test_default_token', schema=flat_social_schema_location,
curated=PurePosixPath(f'{curated}/collection_dlflatsocial-1'), default_token='test_default_token',
incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-1'), curated=PurePosixPath(f'{curated}/collection_dlflatsocial-1'),
), incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-1'),
CollectionRequest(
name='collection_dlflatsocial-2',
schema=flat_social_schema_location,
default_token='test_default_token',
curated=PurePosixPath(f'{curated}/collection_dlflatsocial-2'),
incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-2'),
backend=SQLiteBackendConfig(
type='sqlite',
), ),
use_classes=[ CollectionRequest(
'Organization', name='collection_dlflatsocial-2',
'Person', schema=flat_social_schema_location,
'Project', default_token='test_default_token',
], curated=PurePosixPath(f'{curated}/collection_dlflatsocial-2'),
ignore_classes=[ incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-2'),
'Organization', backend=SQLiteBackendConfig(
'Project', type='sqlite',
], ),
), use_classes=[
]) 'Organization',
'Person',
'Project',
],
ignore_classes=[
'Organization',
'Project',
],
),
]
)
g_default_tokens = [ g_default_tokens = [
TokenRequest( TokenRequest(
@ -152,7 +159,7 @@ g_default_tokens = [
hashed=False, hashed=False,
representation='token-2', representation='token-2',
collections={ collections={
f'collection_2': TokenCollectionConfig( 'collection_2': TokenCollectionConfig(
mode=TokenModes.WRITE_COLLECTION, mode=TokenModes.WRITE_COLLECTION,
incoming_label='in_token-2', incoming_label='in_token-2',
) )
@ -164,7 +171,7 @@ g_default_tokens = [
hashed=False, hashed=False,
representation='token-8', representation='token-8',
collections={ collections={
f'collection_8': TokenCollectionConfig( 'collection_8': TokenCollectionConfig(
mode=TokenModes.WRITE_COLLECTION, mode=TokenModes.WRITE_COLLECTION,
incoming_label='test_user_8', incoming_label='test_user_8',
) )
@ -235,7 +242,7 @@ g_default_tokens = [
mode=TokenModes.WRITE_COLLECTION, mode=TokenModes.WRITE_COLLECTION,
incoming_label='modes', incoming_label='modes',
), ),
} },
), ),
TokenRequest( TokenRequest(
name='Test 0X000 (READ_SUBMISSIONS)', name='Test 0X000 (READ_SUBMISSIONS)',
@ -354,7 +361,8 @@ def fastapi_app_simple(dump_stores_simple):
old_sys_argv = sys.argv old_sys_argv = sys.argv
sys.argv = [ sys.argv = [
'test-runner', 'test-runner',
'--admin-token-hash', hash_token_representation(admin_token), '--admin-token-hash',
hash_token_representation(admin_token),
'--ignore-default-config-file', '--ignore-default-config-file',
str(tmp_path), str(tmp_path),
] ]
@ -429,15 +437,15 @@ def fastapi_client_simple(fastapi_app_simple):
def add_records_to_backend( def add_records_to_backend(
backend: StorageBackend, backend: StorageBackend,
pydantic_module: ModuleType, pydantic_module: ModuleType,
record_infos: list[tuple[str, str, str]], record_infos: list[tuple[str, str, str]],
): ):
for class_name, record_pid, yaml_stream in record_infos: for class_name, record_pid, yaml_stream in record_infos:
json_object = yaml.load(yaml_stream, Loader=yaml.SafeLoader ) json_object = yaml.load(yaml_stream, Loader=yaml.SafeLoader)
assert record_pid == json_object['pid'] assert record_pid == json_object['pid']
backend.add_record( backend.add_record(
iri=resolve_curie(pydantic_module, json_object['pid']), iri=resolve_curie(pydantic_module, json_object['pid']),
class_name=class_name, class_name=class_name,
json_object=json_object, json_object=json_object,
) )

22
dump_things_service/tests/test_auth.py
View file

@ -15,11 +15,7 @@ user_1 = {
'@type': 'user', '@type': 'user',
} }
org_1 = { org_1 = {'id': 1, 'name': 'org_1', '@type': 'org'}
'id': 1,
'name': 'org_1',
'@type': 'org'
}
repo_1 = { repo_1 = {
'id': 3, 'id': 3,
@ -46,10 +42,18 @@ team_3 = json.loads(team_template.format(id=3, action='write'))
def setup_http_server(http_server) -> None: def setup_http_server(http_server) -> None:
for instance in ('1', '2'): for instance in ('1', '2'):
http_server.expect_request(f'/api/v{instance}/user').respond_with_json(user_1) http_server.expect_request(f'/api/v{instance}/user').respond_with_json(user_1)
http_server.expect_request(f'/api/v{instance}/user/teams').respond_with_json([team_1, team_3]) http_server.expect_request(f'/api/v{instance}/user/teams').respond_with_json(
http_server.expect_request(f'/api/v{instance}/orgs/org_1').respond_with_json(org_1) [team_1, team_3]
http_server.expect_request(f'/api/v{instance}/orgs/org_1/teams').respond_with_json([team_1, team_2, team_3]) )
http_server.expect_request(f'/api/v{instance}/repos/org_1/repo_1/teams').respond_with_json([team_1, team_2, team_3]) http_server.expect_request(f'/api/v{instance}/orgs/org_1').respond_with_json(
org_1
)
http_server.expect_request(
f'/api/v{instance}/orgs/org_1/teams'
).respond_with_json([team_1, team_2, team_3])
http_server.expect_request(
f'/api/v{instance}/repos/org_1/repo_1/teams'
).respond_with_json([team_1, team_2, team_3])
@pytest.mark.parametrize('repository', ['repo_1', None]) @pytest.mark.parametrize('repository', ['repo_1', None])

12
dump_things_service/tests/test_basic.py
View file

@ -1,7 +1,3 @@
import pytest # F401
from . import schema_file
from .. import ( from .. import (
HTTP_200_OK, HTTP_200_OK,
HTTP_400_BAD_REQUEST, HTTP_400_BAD_REQUEST,
@ -11,14 +7,13 @@ from .. import (
HTTP_503_SERVICE_UNAVAILABLE, HTTP_503_SERVICE_UNAVAILABLE,
) )
from ..__about__ import __version__ from ..__about__ import __version__
from ..utils import cleaned_json from . import schema_file
from .create_store import ( from .create_store import (
given_name, given_name,
pid, pid,
) )
from .test_utils import basic_write_locations from .test_utils import basic_write_locations
extra_record = { extra_record = {
'schema_type': 'abc:Person', 'schema_type': 'abc:Person',
'pid': 'abc:aaaa', 'pid': 'abc:aaaa',
@ -298,7 +293,7 @@ def test_funky_pid(fastapi_client_simple):
def test_token_store_priority(fastapi_client_simple): def test_token_store_priority(fastapi_client_simple):
test_client, store_dir, _ = fastapi_client_simple test_client, _store_dir, _ = fastapi_client_simple
# Post a record with the same pid as the global store's test record, but # Post a record with the same pid as the global store's test record, but
# with different content. # with different content.
@ -393,7 +388,8 @@ def test_server(fastapi_client_simple):
'classes': test_schema_classes, 'classes': test_schema_classes,
} }
for i in range(1, 9) for i in range(1, 9)
] + [ ]
+ [
{ {
'name': f'collection_dlflatsocial-{i}', 'name': f'collection_dlflatsocial-{i}',
'schema': 'https://concepts.datalad.org/s/flat-social/unreleased.yaml', 'schema': 'https://concepts.datalad.org/s/flat-social/unreleased.yaml',

36
dump_things_service/tests/test_collection_administration.py
View file

@ -6,10 +6,10 @@ from pathlib import (
from starlette.testclient import TestClient from starlette.testclient import TestClient
from dump_things_service import ( from dump_things_service import (
HTTP_201_CREATED,
HTTP_200_OK, HTTP_200_OK,
HTTP_404_NOT_FOUND, HTTP_201_CREATED,
HTTP_401_UNAUTHORIZED, HTTP_401_UNAUTHORIZED,
HTTP_404_NOT_FOUND,
) )
from dump_things_service.abstract_config import ( from dump_things_service.abstract_config import (
GitAuditBackendConfig, GitAuditBackendConfig,
@ -19,10 +19,9 @@ from dump_things_service.abstract_config import (
) )
from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.collection_endpoints import CollectionRequest
from dump_things_service.token_endpoints import ( from dump_things_service.token_endpoints import (
TokenRequest,
AdminTokenRequest, AdminTokenRequest,
TokenRequest,
) )
from dump_things_service.utils import cleaned_json
# String representation of curated- and incoming-path # String representation of curated- and incoming-path
curated = 'admin_test_curated' curated = 'admin_test_curated'
@ -55,7 +54,7 @@ new_token_request = TokenRequest(
}, },
) )
new_admin_token_name='New_Admin_Token' new_admin_token_name = 'New_Admin_Token'
plain_new_admin_token = 'admin-XXX' plain_new_admin_token = 'admin-XXX'
new_admin_token_request = AdminTokenRequest( new_admin_token_request = AdminTokenRequest(
name=new_admin_token_name, name=new_admin_token_name,
@ -64,15 +63,12 @@ new_admin_token_request = AdminTokenRequest(
def _name_in_openapi_paths( def _name_in_openapi_paths(
test_client: TestClient, test_client: TestClient,
name: str, name: str,
) -> bool: ) -> bool:
response = test_client.get('/openapi.json') response = test_client.get('/openapi.json')
open_api = response.json() open_api = response.json()
for path in open_api['paths'].keys(): return any(name in path for path in open_api['paths'])
if name in path:
return True
return False
def test_collection_adding(fastapi_client_simple): def test_collection_adding(fastapi_client_simple):
@ -100,7 +96,9 @@ def test_collection_adding(fastapi_client_simple):
headers={'x-dumpthings-token': admin_token}, headers={'x-dumpthings-token': admin_token},
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
new_collection_config = new_collection_request.model_dump(mode='json', by_alias=True) new_collection_config = new_collection_request.model_dump(
mode='json', by_alias=True
)
del new_collection_config['name'] del new_collection_config['name']
assert response.json() == new_collection_config assert response.json() == new_collection_config
@ -123,7 +121,7 @@ def test_collection_adding(fastapi_client_simple):
'user_id': new_token_request.user_id, 'user_id': new_token_request.user_id,
'collections': new_token_request.model_dump(mode='json')['collections'], 'collections': new_token_request.model_dump(mode='json')['collections'],
'hashed': new_token_request.hashed, 'hashed': new_token_request.hashed,
'representation': new_token_request.representation 'representation': new_token_request.representation,
} }
new_record = { new_record = {
@ -204,7 +202,7 @@ def test_collection_putting(fastapi_client_simple, tmp_path):
path=Path(tmp_path), path=Path(tmp_path),
auto_flush_timeout=2, auto_flush_timeout=2,
) )
] ],
) )
# Check that the collection does not yet exist # Check that the collection does not yet exist
@ -259,7 +257,7 @@ def test_collection_reading(fastapi_client_simple):
# Check that the new admin token is not yet working # Check that the new admin token is not yet working
response = test_client.get( response = test_client.get(
f'/collections', '/collections',
headers={'x-dumpthings-token': admin_token}, headers={'x-dumpthings-token': admin_token},
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
@ -273,7 +271,7 @@ def test_admin_token_management(fastapi_client_simple):
# Check that the new admin token is not yet working # Check that the new admin token is not yet working
response = test_client.get( response = test_client.get(
f'/collections/collection_1', '/collections/collection_1',
headers={'x-dumpthings-token': plain_new_admin_token}, headers={'x-dumpthings-token': plain_new_admin_token},
) )
assert response.status_code == HTTP_401_UNAUTHORIZED assert response.status_code == HTTP_401_UNAUTHORIZED
@ -288,14 +286,14 @@ def test_admin_token_management(fastapi_client_simple):
# Try the new token # Try the new token
response = test_client.get( response = test_client.get(
f'/collections/collection_1', '/collections/collection_1',
headers={'x-dumpthings-token': plain_new_admin_token}, headers={'x-dumpthings-token': plain_new_admin_token},
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
# Check that the token shows up in the token list # Check that the token shows up in the token list
response = test_client.get( response = test_client.get(
f'/admin_tokens', '/admin_tokens',
headers={'x-dumpthings-token': plain_new_admin_token}, headers={'x-dumpthings-token': plain_new_admin_token},
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
@ -310,7 +308,7 @@ def test_admin_token_management(fastapi_client_simple):
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
response = test_client.get( response = test_client.get(
f'/admin_tokens', '/admin_tokens',
headers={'x-dumpthings-token': admin_token}, headers={'x-dumpthings-token': admin_token},
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK

55
dump_things_service/tests/test_config.py
View file

@ -22,13 +22,12 @@ from dump_things_service.exceptions import ConfigError
from dump_things_service.tests import schema_file from dump_things_service.tests import schema_file
from dump_things_service.token_endpoints import TokenRequest from dump_things_service.token_endpoints import TokenRequest
collection_request_pattern = CollectionRequest( collection_request_pattern = CollectionRequest(
name='', name='',
schema=str(schema_file), schema=str(schema_file),
default_token='test_default_token', default_token='test_default_token',
curated=PurePosixPath('curate_dir'), curated=PurePosixPath('curate_dir'),
incoming=PurePosixPath(f'incoming_dir'), incoming=PurePosixPath('incoming_dir'),
) )
@ -36,13 +35,13 @@ def test_illegal_collection_name_detection(fastapi_client_simple):
test_client, _, admin_token = fastapi_client_simple test_client, _, admin_token = fastapi_client_simple
for name in ( for name in (
'collections', 'collections',
'tokens', 'tokens',
'admin_tokens', 'admin_tokens',
dump_things_private_collection_name, dump_things_private_collection_name,
): ):
response = test_client.post( response = test_client.post(
f'/collections', '/collections',
json={ json={
**collection_request_pattern.model_dump(mode='json', by_alias=True), **collection_request_pattern.model_dump(mode='json', by_alias=True),
'name': name, 'name': name,
@ -52,17 +51,19 @@ def test_illegal_collection_name_detection(fastapi_client_simple):
assert response.status_code == HTTP_409_CONFLICT assert response.status_code == HTTP_409_CONFLICT
@pytest.mark.skip(reason='Reuse detection is disabled to support existing old configurations') @pytest.mark.skip(
reason='Reuse detection is disabled to support existing old configurations'
)
def test_collection_dir_reuse_detection(fastapi_client_simple): def test_collection_dir_reuse_detection(fastapi_client_simple):
test_client, _, admin_token = fastapi_client_simple test_client, _, admin_token = fastapi_client_simple
for curated_path, incoming_path in ( for curated_path, incoming_path in (
('curated/collection_1', 'incoming/XXXX'), ('curated/collection_1', 'incoming/XXXX'),
('curated/XXXX', 'incoming/collection_1'), ('curated/XXXX', 'incoming/collection_1'),
('curated/collection_1', 'incoming/collection_2'), ('curated/collection_1', 'incoming/collection_2'),
): ):
response = test_client.post( response = test_client.post(
f'/collections', '/collections',
json={ json={
**collection_request_pattern.model_dump(mode='json', by_alias=True), **collection_request_pattern.model_dump(mode='json', by_alias=True),
'curated': curated_path, 'curated': curated_path,
@ -76,15 +77,17 @@ def test_collection_dir_reuse_detection(fastapi_client_simple):
def test_scanner_error_detection(tmp_path_factory): def test_scanner_error_detection(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp('config_scanner_test') tmp_path = tmp_path_factory.mktemp('config_scanner_test')
config_backend, audit_backend = get_config_backends(tmp_path) config_backend, _audit_backend = get_config_backends(tmp_path)
config_backend.add_record( config_backend.add_record(
iri=dump_things_config_iri, iri=dump_things_config_iri,
class_name='DumpThingsConfig', class_name='DumpThingsConfig',
json_object={'pid': dump_things_config_iri} json_object={'pid': dump_things_config_iri},
) )
md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest() md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest()
config_file_path = config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' config_file_path = (
config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml'
)
config_file_path.write_text('collections: ::: -\n sdsdfsdf: xxx') config_file_path.write_text('collections: ::: -\n sdsdfsdf: xxx')
with pytest.raises(ConfigError): with pytest.raises(ConfigError):
read_config(tmp_path, force_reload=True) read_config(tmp_path, force_reload=True)
@ -93,15 +96,17 @@ def test_scanner_error_detection(tmp_path_factory):
def test_structure_error_detection(tmp_path_factory): def test_structure_error_detection(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp('config_scanner_test') tmp_path = tmp_path_factory.mktemp('config_scanner_test')
config_backend, audit_backend = get_config_backends(tmp_path) config_backend, _audit_backend = get_config_backends(tmp_path)
config_backend.add_record( config_backend.add_record(
iri=dump_things_config_iri, iri=dump_things_config_iri,
class_name='DumpThingsConfig', class_name='DumpThingsConfig',
json_object={'pid': dump_things_config_iri} json_object={'pid': dump_things_config_iri},
) )
md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest() md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest()
config_file_path = config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' config_file_path = (
config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml'
)
config_file_path.write_text('type: 1\n') config_file_path.write_text('type: 1\n')
with pytest.raises(ConfigError): with pytest.raises(ConfigError):
read_config(tmp_path, force_reload=True) read_config(tmp_path, force_reload=True)
@ -135,7 +140,7 @@ def test_missing_incoming_detection(fastapi_client_simple):
mode=TokenModes.CURATOR, mode=TokenModes.CURATOR,
incoming_label='', incoming_label='',
) )
} },
) )
# Check that a write token for a collection without incoming path cannot # Check that a write token for a collection without incoming path cannot
@ -155,7 +160,9 @@ def test_missing_incoming_detection(fastapi_client_simple):
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
# Add a collection with incoming path # Add a collection with incoming path
collection_request.incoming = PurePosixPath('missing_incoming_detection_test_incoming') collection_request.incoming = PurePosixPath(
'missing_incoming_detection_test_incoming'
)
response = test_client.post( response = test_client.post(
'/collections', '/collections',
json=collection_request.model_dump(mode='json', by_alias=True), json=collection_request.model_dump(mode='json', by_alias=True),
@ -173,9 +180,11 @@ def test_missing_incoming_detection(fastapi_client_simple):
assert response.status_code == HTTP_406_NOT_ACCEPTABLE assert response.status_code == HTTP_406_NOT_ACCEPTABLE
# Check that a write token for a collection with an incoming path can be created # Check that a write token for a collection with an incoming path can be created
token_request.collections['missing_incoming_detection_test'] = TokenCollectionConfig( token_request.collections['missing_incoming_detection_test'] = (
mode=TokenModes.CURATOR, TokenCollectionConfig(
incoming_label='test_incoming_label', mode=TokenModes.CURATOR,
incoming_label='test_incoming_label',
)
) )
response = test_client.post( response = test_client.post(
'/tokens', '/tokens',

17
dump_things_service/tests/test_curated.py
View file

@ -1,17 +1,17 @@
from __future__ import annotations from __future__ import annotations
import pytest
import time import time
import yaml
from itertools import count from itertools import count
import pytest
import yaml
from dump_things_service import ( from dump_things_service import (
HTTP_200_OK, HTTP_200_OK,
HTTP_404_NOT_FOUND, HTTP_404_NOT_FOUND,
) )
from dump_things_service.instance_state import get_instance_state from dump_things_service.instance_state import get_instance_state
delete_record = { delete_record = {
'schema_type': 'abc:Person', 'schema_type': 'abc:Person',
'pid': 'abc:delete-me', 'pid': 'abc:delete-me',
@ -19,8 +19,8 @@ delete_record = {
} }
@pytest.mark.parametrize('paginate', ('', 'p/')) @pytest.mark.parametrize('paginate', ['', 'p/'])
@pytest.mark.parametrize('class_name', ('', 'Person')) @pytest.mark.parametrize('class_name', ['', 'Person'])
def test_read_curated_records( def test_read_curated_records(
fastapi_client_simple, fastapi_client_simple,
paginate, paginate,
@ -54,10 +54,6 @@ def test_read_curated_records(
assert len(json_object) == count assert len(json_object) == count
pytest.mark.parametrize(
'pid',
('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'),
)
def test_read_curated_records_by_pid(fastapi_client_simple): def test_read_curated_records_by_pid(fastapi_client_simple):
test_client, _, _ = fastapi_client_simple test_client, _, _ = fastapi_client_simple
@ -185,5 +181,6 @@ def test_audit_backend_auto_flush(fastapi_client_simple):
break break
i += 1 i += 1
if i == 10: if i == 10:
raise ValueError(f'auto flush did not trigger within 10 seconds') msg = 'auto flush did not trigger within 10 seconds'
raise ValueError(msg)
time.sleep(1) time.sleep(1)

37
dump_things_service/tests/test_extract_inline.py
View file

@ -113,7 +113,10 @@ empty_inlined_json_record = cleaned_json(dataclasses.asdict(empty_inlined_object
tree = ( tree = (
('dlflatsocial:test_extract_1', ('dlflatsocial:test_extract_1_1', 'dlflatsocial:test_extract_1_2')), (
'dlflatsocial:test_extract_1',
('dlflatsocial:test_extract_1_1', 'dlflatsocial:test_extract_1_2'),
),
('dlflatsocial:test_extract_1_1', ('dlflatsocial:test_extract_1_1_1',)), ('dlflatsocial:test_extract_1_1', ('dlflatsocial:test_extract_1_1_1',)),
('dlflatsocial:test_extract_1_2', ()), ('dlflatsocial:test_extract_1_2', ()),
('dlflatsocial:test_extract_1_1_1', ()), ('dlflatsocial:test_extract_1_1_1', ()),
@ -181,10 +184,10 @@ def test_inline_extraction_locally():
store = ModelStore( store = ModelStore(
schema=str(schema_path), schema=str(schema_path),
backend=None, backend=None,
tags = { tags={
'id': 'abc:id', 'id': 'abc:id',
'time': 'abc:time', 'time': 'abc:time',
} },
) )
store.model = MockedModule() store.model = MockedModule()
records = store.extract_inlined(inlined_object) records = store.extract_inlined(inlined_object)
@ -216,7 +219,7 @@ def test_dont_extract_empty_things_locally():
tags={ tags={
'id': 'https://id', 'id': 'https://id',
'time': 'https://time', 'time': 'https://time',
} },
) )
store.model = MockedModule() store.model = MockedModule()
records = store.extract_inlined(empty_inlined_object) records = store.extract_inlined(empty_inlined_object)
@ -257,7 +260,10 @@ def test_inline_extraction_on_service(fastapi_client_simple):
# Check that individual record classes were recognized # Check that individual record classes were recognized
for class_name, pids in ( for class_name, pids in (
('Person', ('dlflatsocial:test_extract_1', 'dlflatsocial:test_extract_1_1')), (
'Person',
('dlflatsocial:test_extract_1', 'dlflatsocial:test_extract_1_1'),
),
('Agent', ('dlflatsocial:test_extract_1_1_1',)), ('Agent', ('dlflatsocial:test_extract_1_1_1',)),
('InstantaneousEvent', ('dlflatsocial:test_extract_1_2',)), ('InstantaneousEvent', ('dlflatsocial:test_extract_1_2',)),
): ):
@ -301,7 +307,10 @@ def test_inline_ttl_processing(fastapi_client_simple):
# Check that individual record classes were recognized # Check that individual record classes were recognized
for class_name, pids in ( for class_name, pids in (
('Person', ('dlflatsocial:test_ttl_inline_1', 'dlflatsocial:test_ttl_inline_1_1')), (
'Person',
('dlflatsocial:test_ttl_inline_1', 'dlflatsocial:test_ttl_inline_1_1'),
),
('Agent', ('dlflatsocial:test_ttl_inline_1_1_1',)), ('Agent', ('dlflatsocial:test_ttl_inline_1_1_1',)),
('InstantaneousEvent', ('dlflatsocial:test_ttl_inline_1_2',)), ('InstantaneousEvent', ('dlflatsocial:test_ttl_inline_1_2',)),
): ):
@ -339,7 +348,7 @@ def _check_result_json(
# That breaks the tests. They assume that Person.relations has range Thing. # That breaks the tests. They assume that Person.relations has range Thing.
@pytest.mark.xfail @pytest.mark.xfail
def test_dont_extract_empty_things_on_service(fastapi_client_simple): def test_dont_extract_empty_things_on_service(fastapi_client_simple):
test_client, store = fastapi_client_simple test_client, _store = fastapi_client_simple
for i in range(1, 3): for i in range(1, 3):
# Deposit JSON record # Deposit JSON record
@ -352,7 +361,7 @@ def test_dont_extract_empty_things_on_service(fastapi_client_simple):
def test_store_things(fastapi_client_simple): def test_store_things(fastapi_client_simple):
test_client, store, _ = fastapi_client_simple test_client, _store, _ = fastapi_client_simple
simple_thing = { simple_thing = {
'pid': 'http://test.simple.thing/1', 'pid': 'http://test.simple.thing/1',
@ -375,7 +384,7 @@ def test_store_things(fastapi_client_simple):
def test_store_complex_things(fastapi_client_simple): def test_store_complex_things(fastapi_client_simple):
test_client, store, _ = fastapi_client_simple test_client, _store, _ = fastapi_client_simple
complex_thing = { complex_thing = {
'pid': 'http://test.complex.thing/1', 'pid': 'http://test.complex.thing/1',
@ -386,9 +395,9 @@ def test_store_complex_things(fastapi_client_simple):
'http://test.complex.thing/1.1.1': { 'http://test.complex.thing/1.1.1': {
'pid': 'http://test.complex.thing/1.1.1', 'pid': 'http://test.complex.thing/1.1.1',
} }
} },
} }
} },
} }
# Deposit JSON record # Deposit JSON record
@ -402,9 +411,9 @@ def test_store_complex_things(fastapi_client_simple):
# Try to read individual extracted elements # Try to read individual extracted elements
for pid in ( for pid in (
'http://test.complex.thing/1', 'http://test.complex.thing/1',
'http://test.complex.thing/1.1', 'http://test.complex.thing/1.1',
'http://test.complex.thing/1.1.1', 'http://test.complex.thing/1.1.1',
): ):
response = test_client.get( response = test_client.get(
f'/collection_1/record?pid={pid}', f'/collection_1/record?pid={pid}',

10
dump_things_service/tests/test_ifabsent_patch.py
View file

@ -6,7 +6,6 @@ import linkml.generators.common.ifabsent_processor as if_abs_proc
import dump_things_service.patches.ifabsent_processing import dump_things_service.patches.ifabsent_processing
# Path to a local simple test schema # Path to a local simple test schema
schema_dir = Path(__file__).parent / 'assets' schema_dir = Path(__file__).parent / 'assets'
@ -17,10 +16,11 @@ def _original_uri_for(self, s: str) -> str:
def test_ifabsent_patch(): def test_ifabsent_patch():
# Patch in the faulty, original code and check for its result # Patch in the faulty, original code and check for its result
if_abs_proc.IfAbsentProcessor._uri_for = _original_uri_for if_abs_proc.IfAbsentProcessor._uri_for = _original_uri_for
gen1 = linkml.generators.PydanticGenerator(str(schema_dir / 'schema-ifabsent-error.yaml')) gen1 = linkml.generators.PydanticGenerator(
str(schema_dir / 'schema-ifabsent-error.yaml')
)
x = gen1.serialize() x = gen1.serialize()
assert 'default=XSD["04fa4r544"]' in x assert 'default=XSD["04fa4r544"]' in x
@ -28,6 +28,8 @@ def test_ifabsent_patch():
reload(dump_things_service.patches.ifabsent_processing) reload(dump_things_service.patches.ifabsent_processing)
# Check for proper code generation # Check for proper code generation
gen2 = linkml.generators.PydanticGenerator(str(schema_dir / 'schema-ifabsent-error.yaml')) gen2 = linkml.generators.PydanticGenerator(
str(schema_dir / 'schema-ifabsent-error.yaml')
)
y = gen2.serialize() y = gen2.serialize()
assert 'XSD' not in y assert 'XSD' not in y

50
dump_things_service/tests/test_incoming.py
View file

@ -30,6 +30,7 @@ def test_incoming_labels(fastapi_client_simple):
zones_filled = False zones_filled = False
def fill_zones(test_client): def fill_zones(test_client):
global zones_filled global zones_filled
@ -53,33 +54,33 @@ def fill_zones(test_client):
json={ json={
'pid': f'abc:test_incoming-collection_{collection_id}-{token}', 'pid': f'abc:test_incoming-collection_{collection_id}-{token}',
'given_name': f'collection_{collection_id}-{token}', 'given_name': f'collection_{collection_id}-{token}',
} },
) )
assert result.status_code == HTTP_200_OK assert result.status_code == HTTP_200_OK
zones_filled = True zones_filled = True
@pytest.mark.parametrize('paginate', ('', 'p/')) @pytest.mark.parametrize('paginate', ['', 'p/'])
@pytest.mark.parametrize('class_name', ('', 'Person')) @pytest.mark.parametrize('class_name', ['', 'Person'])
def test_read_incoming_records( def test_read_incoming_records(
fastapi_client_simple, fastapi_client_simple,
paginate: str, paginate: str,
class_name: str, class_name: str,
): ):
test_client, _, _ = fastapi_client_simple test_client, _, _ = fastapi_client_simple
fill_zones(test_client) fill_zones(test_client)
for collection_id, labels in ( for collection_id, labels in (
(1, ['modes', 'admin_1', 'in_token_1']), (1, ['modes', 'admin_1', 'in_token_1']),
(2, ['in_token-2', 'admin_2']), (2, ['in_token-2', 'admin_2']),
(3, ['admin_3']), (3, ['admin_3']),
(4, ['admin_4']), (4, ['admin_4']),
(5, ['admin_common']), (5, ['admin_common']),
(6, ['admin_common']), (6, ['admin_common']),
(7, ['admin_common']), (7, ['admin_common']),
(8, ['modes', 'test_user_8', 'admin_common']), (8, ['modes', 'test_user_8', 'admin_common']),
): ):
# Check that all incoming zones are reached # Check that all incoming zones are reached
for label in labels: for label in labels:
@ -87,7 +88,9 @@ def test_read_incoming_records(
f'/collection_{collection_id}/incoming/{label}/records/{paginate}{class_name}', f'/collection_{collection_id}/incoming/{label}/records/{paginate}{class_name}',
headers={'x-dumpthings-token': 'token_curator'}, headers={'x-dumpthings-token': 'token_curator'},
) )
assert response.status_code == HTTP_200_OK, f'failed on collection: {collection_id}, label: {label}, class: {class_name}' assert response.status_code == HTTP_200_OK, (
f'failed on collection: {collection_id}, label: {label}, class: {class_name}'
)
# We don't know the exact number of entries in each zone, because # We don't know the exact number of entries in each zone, because
# it depends on the tests that ran before. # it depends on the tests that ran before.
@ -103,22 +106,15 @@ def test_read_incoming_records(
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
json_object = response.json() json_object = response.json()
if 'items' in json_object: result = json_object['items'] if 'items' in json_object else json_object
result = json_object['items']
else:
result = json_object
matching = [ matching = [
json_object json_object for json_object in result if json_object['pid'] == pattern
for json_object in result
if json_object['pid'] == pattern
] ]
assert len(matching) == expected_length, f'did not find {expected_length} record: collection_{collection_id}, {label}, {result}' assert len(matching) == expected_length, (
f'did not find {expected_length} record: collection_{collection_id}, {label}, {result}'
)
pytest.mark.parametrize(
'pid',
('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'),
)
def test_read_incoming_records_by_pid(fastapi_client_simple): def test_read_incoming_records_by_pid(fastapi_client_simple):
test_client, _, _ = fastapi_client_simple test_client, _, _ = fastapi_client_simple

2
dump_things_service/tests/test_modes.py
View file

@ -50,7 +50,7 @@ def verify_modes(
def test_token_modes(fastapi_client_simple): def test_token_modes(fastapi_client_simple):
test_client, store_dir, _ = fastapi_client_simple test_client, _store_dir, _ = fastapi_client_simple
# Post a record to incoming of collections `collection_1`. We use it to # Post a record to incoming of collections `collection_1`. We use it to
# validate read/write permissions on class-base # validate read/write permissions on class-base

2
dump_things_service/tests/test_roundtrip.py
View file

@ -1,5 +1,5 @@
import freezegun import freezegun
import pytest # noqa F401 import pytest # noqa: F401
from .. import HTTP_200_OK from .. import HTTP_200_OK
from ..utils import cleaned_json from ..utils import cleaned_json

15
dump_things_service/tests/test_roundtrip_flatsocial.py
View file

@ -1,6 +1,5 @@
import pytest # noqa F401
import freezegun import freezegun
import pytest # noqa: F401
from .. import HTTP_200_OK from .. import HTTP_200_OK
from ..utils import cleaned_json from ..utils import cleaned_json
@ -144,7 +143,9 @@ def test_ttl_json_ttl_dlflatsocial(fastapi_client_simple):
}, },
data=ttl_input_record, data=ttl_input_record,
) )
assert response.status_code == HTTP_200_OK, 'Response content: ' + response.content.decode() assert response.status_code == HTTP_200_OK, (
'Response content: ' + response.content.decode()
)
# Retrieve JSON records # Retrieve JSON records
response = test_client.get( response = test_client.get(
@ -172,8 +173,12 @@ def test_ttl_json_ttl_dlflatsocial(fastapi_client_simple):
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK
assert ( assert (
response.text.strip() response.text.strip()
== ttl_output_record_a.replace('dlflatsocial:test_john_ttl', new_json_pid).strip() == ttl_output_record_a.replace(
'dlflatsocial:test_john_ttl', new_json_pid
).strip()
) or ( ) or (
response.text.strip() response.text.strip()
== ttl_output_record_b.replace('dlflatsocial:test_john_ttl', new_json_pid).strip() == ttl_output_record_b.replace(
'dlflatsocial:test_john_ttl', new_json_pid
).strip()
) )

11
dump_things_service/tests/test_token_endpoints.py
View file

@ -11,14 +11,11 @@ def test_token_creation(fastapi_client_simple):
'user_id': 'u_a', 'user_id': 'u_a',
'representation': '8bb6805ff10bcb1c2ca49dcd4bfef94d', 'representation': '8bb6805ff10bcb1c2ca49dcd4bfef94d',
'collections': { 'collections': {
'collection_1': { 'collection_1': {'mode': 'WRITE_COLLECTION', 'incoming_label': 'i_a'}
'mode': 'WRITE_COLLECTION', },
'incoming_label': 'i_a'
}
}
} }
# Create a token eith name 'a' # Create a token with name 'a'
response = test_client.post( response = test_client.post(
'/tokens', '/tokens',
headers={'x-dumpthings-token': admin_token}, headers={'x-dumpthings-token': admin_token},
@ -34,7 +31,7 @@ def test_token_creation(fastapi_client_simple):
) )
assert response.status_code == HTTP_409_CONFLICT assert response.status_code == HTTP_409_CONFLICT
# Try to create another token eith name 'b' and the same representation # Try to create another token with name 'b' and the same representation
# as 'a', should result in a 4ß9-error # as 'a', should result in a 4ß9-error
json_record['name'] = 'b' json_record['name'] = 'b'
response = test_client.post( response = test_client.post(

7
dump_things_service/tests/test_unicode.py
View file

@ -2,7 +2,6 @@ from pathlib import Path
from .. import HTTP_200_OK from .. import HTTP_200_OK
# Path to a local simple test schema # Path to a local simple test schema
schema_file = Path(__file__).parent / 'testschema.yaml' schema_file = Path(__file__).parent / 'testschema.yaml'
@ -31,9 +30,9 @@ def test_unicode_iri(fastapi_client_simple):
response = test_client.post( response = test_client.post(
'/collection_1/record/Person', '/collection_1/record/Person',
headers={'x-dumpthings-token': 'token-1'}, headers={'x-dumpthings-token': 'token-1'},
json = { json={
'pid': 'https://en.wikipedia.org/wiki/Universita_degli_Studi_eCampus', 'pid': 'https://en.wikipedia.org/wiki/Universita_degli_Studi_eCampus',
'given_name': 'Università degli Studi eCampus (Italy)', 'given_name': 'Università degli Studi eCampus (Italy)', # codespell:ignore
} },
) )
assert response.status_code == HTTP_200_OK assert response.status_code == HTTP_200_OK

6
dump_things_service/tests/test_validate.py
View file

@ -1,9 +1,11 @@
from dump_things_service import HTTP_422_UNPROCESSABLE_CONTENT from dump_things_service import HTTP_422_UNPROCESSABLE_CONTENT
json_records = [ json_records = [
({'name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT), ({'name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT),
({'given_name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT), (
{'given_name': 'Henry', 'pid': 'unknown_prefix:henry'},
HTTP_422_UNPROCESSABLE_CONTENT,
),
({'given_name': 'Henry', 'pid': 'xyz:henry'}, 200), ({'given_name': 'Henry', 'pid': 'xyz:henry'}, 200),
] ]

36
dump_things_service/tests/test_web_interface.py
View file

@ -14,15 +14,15 @@ pids = ('', '--------', '&&&&&', 'abc', 'abc&', 'abc&format=ttl')
@pytest.mark.parametrize( @pytest.mark.parametrize(
'collection_name,class_name,query,format_name', # noqa PT006 'collection_name,class_name,query,format_name', # noqa: PT006
tuple(product(*(collection_names, class_names, queries, format_names))), tuple(product(*(collection_names, class_names, queries, format_names))),
) )
def test_web_interface_post_errors( def test_web_interface_post_errors(
fastapi_client_simple, fastapi_client_simple,
collection_name, collection_name,
class_name, class_name,
query, query,
format_name, format_name,
): ):
"""Check that no internal server error occurs with weird input""" """Check that no internal server error occurs with weird input"""
test_client, _, _ = fastapi_client_simple test_client, _, _ = fastapi_client_simple
@ -35,15 +35,15 @@ def test_web_interface_post_errors(
@pytest.mark.parametrize( @pytest.mark.parametrize(
'collection_name,class_name,query,format_name', # noqa PT006 'collection_name,class_name,query,format_name', # noqa: PT006
tuple(product(*(collection_names, class_names, queries, format_names))), tuple(product(*(collection_names, class_names, queries, format_names))),
) )
def test_web_interface_get_class_errors( def test_web_interface_get_class_errors(
fastapi_client_simple, fastapi_client_simple,
collection_name, collection_name,
class_name, class_name,
query, query,
format_name, format_name,
): ):
"""Check that no internal server error occurs with weird input""" """Check that no internal server error occurs with weird input"""
test_client, _, _ = fastapi_client_simple test_client, _, _ = fastapi_client_simple
@ -60,15 +60,15 @@ def test_web_interface_get_class_errors(
@pytest.mark.parametrize( @pytest.mark.parametrize(
'collection_name,pid,query,format_name', # noqa PT006 'collection_name,pid,query,format_name', # noqa: PT006
tuple(product(*(collection_names, pids, queries, format_names))), tuple(product(*(collection_names, pids, queries, format_names))),
) )
def test_web_interface_get_pid_errors( def test_web_interface_get_pid_errors(
fastapi_client_simple, fastapi_client_simple,
collection_name, collection_name,
pid, pid,
query, query,
format_name, format_name,
): ):
"""Check that no internal server error occurs with weird input""" """Check that no internal server error occurs with weird input"""
test_client, _, _ = fastapi_client_simple test_client, _, _ = fastapi_client_simple

79
dump_things_service/token_endpoints.py
View file

@ -1,6 +1,7 @@
import logging import logging
import random import random
import re import re
from typing import Annotated
from urllib.parse import quote from urllib.parse import quote
from fastapi import ( from fastapi import (
@ -30,12 +31,11 @@ from dump_things_service.abstract_config import (
) )
from dump_things_service.admin import authenticate_admin from dump_things_service.admin import authenticate_admin
from dump_things_service.api_key import api_key_header_scheme from dump_things_service.api_key import api_key_header_scheme
from dump_things_service.instance_state import get_instance_state
from dump_things_service.exceptions import ConfigError from dump_things_service.exceptions import ConfigError
from dump_things_service.instance_state import get_instance_state
from dump_things_service.manifest import manifest_configuration from dump_things_service.manifest import manifest_configuration
from dump_things_service.utils import wrap_http_exception from dump_things_service.utils import wrap_http_exception
logger = logging.getLogger('dump_things_service') logger = logging.getLogger('dump_things_service')
router = APIRouter() router = APIRouter()
@ -71,11 +71,10 @@ def get_token_parts(token: str) -> list[str]:
status_code=HTTP_201_CREATED, status_code=HTTP_201_CREATED,
) )
async def create_token( async def create_token(
response: Response, response: Response,
body: TokenRequest, body: TokenRequest,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> TokenRequest: ) -> TokenRequest:
token_request = create_or_replace_token(body, api_key, allow_replace=False) token_request = create_or_replace_token(body, api_key, allow_replace=False)
response.headers['Location'] = f'/tokens/{quote(body.name)}' response.headers['Location'] = f'/tokens/{quote(body.name)}'
return token_request return token_request
@ -88,23 +87,21 @@ async def create_token(
status_code=HTTP_201_CREATED, status_code=HTTP_201_CREATED,
) )
async def replace_token( async def replace_token(
response: Response, response: Response,
body: TokenRequest, body: TokenRequest,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> TokenRequest: ) -> TokenRequest:
token_request = create_or_replace_token(body, api_key, allow_replace=True) token_request = create_or_replace_token(body, api_key, allow_replace=True)
response.headers['Location'] = f'/tokens/{quote(body.name)}' response.headers['Location'] = f'/tokens/{quote(body.name)}'
return token_request return token_request
def create_or_replace_token( def create_or_replace_token(
body: TokenRequest, body: TokenRequest,
api_key: str, api_key: str,
*, *,
allow_replace: bool, allow_replace: bool,
) -> TokenRequest: ) -> TokenRequest:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = read_config(store_path=instance_state.store_path) abstract_config = read_config(store_path=instance_state.store_path)
@ -118,7 +115,7 @@ def create_or_replace_token(
) )
# Ensure that all specified collections and modes exist # Ensure that all specified collections and modes exist
for collection_name, token_collection_info in body.collections.items(): for collection_name, token_collection_info in body.collections.items():
if collection_name not in abstract_config.collections: if collection_name not in abstract_config.collections:
detail = f"No such collection: '{collection_name}'." detail = f"No such collection: '{collection_name}'."
raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=detail) raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=detail)
@ -126,12 +123,11 @@ def create_or_replace_token(
# Check that incoming areas are defined if the token allows writing. # Check that incoming areas are defined if the token allows writing.
token_permissions = get_token_permissions(token_collection_info.mode) token_permissions = get_token_permissions(token_collection_info.mode)
if token_permissions.incoming_write or token_permissions.zones_access: if token_permissions.incoming_write or token_permissions.zones_access:
# Check for incoming definition in collection config # Check for incoming definition in collection config
collection_info = abstract_config.collections[collection_name] collection_info = abstract_config.collections[collection_name]
if not collection_info.incoming: if not collection_info.incoming:
detail = ( detail = (
f"Cannot add token with write access to collection " f'Cannot add token with write access to collection '
f"'{collection_name}' without `incoming`." f"'{collection_name}' without `incoming`."
) )
raise HTTPException( raise HTTPException(
@ -154,7 +150,7 @@ def create_or_replace_token(
token_representation=body.representation, token_representation=body.representation,
) )
if existing_token_info: if existing_token_info:
detail= f"Token with identical representation already exists." detail = 'Token with identical representation already exists.'
raise HTTPException(status_code=HTTP_409_CONFLICT, detail=detail) raise HTTPException(status_code=HTTP_409_CONFLICT, detail=detail)
else: else:
# Generate a random representation that does not yet exist. # Generate a random representation that does not yet exist.
@ -203,9 +199,8 @@ def create_or_replace_token(
name='Get existing tokens', name='Get existing tokens',
) )
async def get_tokens( async def get_tokens(
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> list[TokenRequest]: ) -> list[TokenRequest]:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = read_config(store_path=instance_state.store_path) abstract_config = read_config(store_path=instance_state.store_path)
@ -229,10 +224,9 @@ async def get_tokens(
name='Get token by name', name='Get token by name',
) )
async def get_token_with_name( async def get_token_with_name(
token_name: str, token_name: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> TokenRequest: ) -> TokenRequest:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -259,10 +253,9 @@ async def get_token_with_name(
name='Delete token with name', name='Delete token with name',
) )
async def delete_token_with_name( async def delete_token_with_name(
token_name: str, token_name: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -294,8 +287,8 @@ async def delete_token_with_name(
status_code=HTTP_201_CREATED, status_code=HTTP_201_CREATED,
) )
async def create_admin_token( async def create_admin_token(
body: AdminTokenRequest, body: AdminTokenRequest,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
return create_or_replace_admin_token(body, api_key, allow_replace=False) return create_or_replace_admin_token(body, api_key, allow_replace=False)
@ -307,17 +300,17 @@ async def create_admin_token(
status_code=HTTP_201_CREATED, status_code=HTTP_201_CREATED,
) )
async def replace_admin_token( async def replace_admin_token(
body: AdminTokenRequest, body: AdminTokenRequest,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
return create_or_replace_admin_token(body, api_key, allow_replace=True) return create_or_replace_admin_token(body, api_key, allow_replace=True)
def create_or_replace_admin_token( def create_or_replace_admin_token(
body: AdminTokenRequest, body: AdminTokenRequest,
api_key: str, api_key: str,
*, *,
allow_replace: bool, allow_replace: bool,
): ):
# Check for conflicting token-name # Check for conflicting token-name
if body.name == '__bootstrap__': if body.name == '__bootstrap__':
@ -328,11 +321,11 @@ def create_or_replace_admin_token(
# Check for token content # Check for token content
if not body.representation: if not body.representation:
detail='Empty administrator token is not allowed' detail = 'Empty administrator token is not allowed'
raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail)
if not hash_matcher.match(body.representation.strip()): if not hash_matcher.match(body.representation.strip()):
detail='Hashed token is not a 64-digits hex-number' detail = 'Hashed token is not a 64-digits hex-number'
raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail)
instance_state = get_instance_state() instance_state = get_instance_state()
@ -369,7 +362,7 @@ def create_or_replace_admin_token(
name='Get admin token names', name='Get admin token names',
) )
async def get_admin_token( async def get_admin_token(
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
) -> list[dict]: ) -> list[dict]:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = read_config(store_path=instance_state.store_path) abstract_config = read_config(store_path=instance_state.store_path)
@ -377,10 +370,7 @@ async def get_admin_token(
authenticate_admin(instance_state, abstract_config, api_key) authenticate_admin(instance_state, abstract_config, api_key)
return [ return [
{ {'name': token_name, **(token_value.model_dump(mode='json', by_alias=True))}
'name': token_name,
**(token_value.model_dump(mode='json', by_alias=True))
}
for token_name, token_value in abstract_config.admin_tokens.items() for token_name, token_value in abstract_config.admin_tokens.items()
] + ( ] + (
[] []
@ -400,10 +390,9 @@ async def get_admin_token(
name='Delete admin token with name', name='Delete admin token with name',
) )
async def delete_admin_token( async def delete_admin_token(
token_name: str, token_name: str,
api_key: str = Depends(api_key_header_scheme), api_key: Annotated[str, Depends(api_key_header_scheme)],
): ):
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = read_config(store_path=instance_state.store_path) abstract_config = read_config(store_path=instance_state.store_path)

191
dump_things_service/utils.py
View file

@ -6,6 +6,7 @@ To speed up processing, multiple indices could be introduced, e.g.:
- token representation -> token name - token representation -> token name
""" """
from __future__ import annotations from __future__ import annotations
import logging import logging
@ -31,12 +32,10 @@ from dump_things_service.abstract_config import (
Configuration, Configuration,
TokenModes, TokenModes,
TokenPermission, TokenPermission,
mode_mapping,
check_collection, check_collection,
get_collection_config_by_name, get_collection_config_by_name,
get_default_token_config,
get_mapping_function_by_name, get_mapping_function_by_name,
get_token_config_for_representation_and_collection, mode_mapping,
) )
from dump_things_service.auth import ( from dump_things_service.auth import (
AuthenticationError, AuthenticationError,
@ -83,7 +82,7 @@ def cleaned_json(data: JSON, remove_keys: tuple[str, ...] = ('@type',)) -> JSON:
return { return {
key: cleaned_json(value, remove_keys) key: cleaned_json(value, remove_keys)
for key, value in data.items() for key, value in data.items()
if key not in remove_keys and data[key] is not None if key not in remove_keys and value is not None
} }
return data return data
@ -97,7 +96,7 @@ def combine_ttl(documents: list[str]) -> str:
def wrap_http_exception( def wrap_http_exception(
exception_class: type[BaseException] = ValueError, exception_class: type[BaseException] = ValueError,
status_code: int = HTTP_400_BAD_REQUEST, status_code: int = HTTP_400_BAD_REQUEST,
header: str = '' header: str = '',
): ):
"""Wrap exceptions of class `exception_class` into HTTP exceptions""" """Wrap exceptions of class `exception_class` into HTTP exceptions"""
try: try:
@ -110,12 +109,11 @@ def wrap_http_exception(
def join_default_token_permissions( def join_default_token_permissions(
abstract_configuration: Configuration, abstract_configuration: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
permissions: TokenPermission, permissions: TokenPermission,
collection: str, collection: str,
) -> TokenPermission: ) -> TokenPermission:
result = permissions.model_copy() result = permissions.model_copy()
# Get the default token name. If a default token is not defined, return # Get the default token name. If a default token is not defined, return
@ -134,46 +132,41 @@ def join_default_token_permissions(
if collection not in abstract_configuration.tokens[default_token_name].collections: if collection not in abstract_configuration.tokens[default_token_name].collections:
return result return result
default_token_mode = abstract_configuration.tokens[default_token_name].collections[collection].mode default_token_mode = (
abstract_configuration.tokens[default_token_name].collections[collection].mode
)
default_token_permissions = mode_mapping[TokenModes(default_token_mode)] default_token_permissions = mode_mapping[TokenModes(default_token_mode)]
result.curated_read = ( result.curated_read = (
permissions.curated_read | default_token_permissions.curated_read permissions.curated_read | default_token_permissions.curated_read
) )
result.incoming_read = ( result.incoming_read = (
permissions.incoming_read | default_token_permissions.incoming_read permissions.incoming_read | default_token_permissions.incoming_read
) )
result.incoming_write = ( result.incoming_write = (
permissions.incoming_write | default_token_permissions.incoming_write permissions.incoming_write | default_token_permissions.incoming_write
) )
return result return result
def get_on_disk_labels( def get_on_disk_labels(
store_path: Path, store_path: Path,
abstract_config: Configuration, abstract_config: Configuration,
collection: str, collection: str,
) -> set[str]: ) -> set[str]:
check_collection(abstract_config, collection) check_collection(abstract_config, collection)
incoming_path = ( incoming_path = store_path / abstract_config.collections[collection].incoming
store_path / abstract_config.collections[collection].incoming
)
if not incoming_path or not incoming_path.exists(): if not incoming_path or not incoming_path.exists():
return set() return set()
return { return {path.name for path in incoming_path.iterdir() if path.is_dir()}
path.name
for path in incoming_path.iterdir()
if path.is_dir()
}
def authenticate_token( def authenticate_token(
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
token_representation: str, token_representation: str,
) -> AuthenticationInfo: ) -> AuthenticationInfo:
# Try to authenticate the token with the authentication providers that # Try to authenticate the token with the authentication providers that
# are associated with the collection. # are associated with the collection.
auth_info = None auth_info = None
@ -206,9 +199,9 @@ def authenticate_token(
def get_default_token_auth_info( def get_default_token_auth_info(
abstract_config: Configuration, abstract_config: Configuration,
collection_name: str, collection_name: str,
token_name: str, token_name: str,
) -> AuthenticationInfo: ) -> AuthenticationInfo:
token_config = abstract_config.tokens[token_name] token_config = abstract_config.tokens[token_name]
collection_info = token_config.collections.get(collection_name) collection_info = token_config.collections.get(collection_name)
@ -220,20 +213,19 @@ def get_default_token_auth_info(
) )
return AuthenticationInfo( return AuthenticationInfo(
token_permission=mode_mapping[TokenModes(collection_info.mode)], token_permission=mode_mapping[TokenModes(collection_info.mode)],
user_id = token_config.user_id, user_id=token_config.user_id,
incoming_label = collection_info.incoming_label, incoming_label=collection_info.incoming_label,
) )
def get_token_store( def get_token_store(
abstract_config: Configuration, abstract_config: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
token_representation: str | None, token_representation: str | None,
*, *,
is_token_name: bool = False, is_token_name: bool = False,
) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]: ) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]:
# If a token representation is provided, try to authenticate the token # If a token representation is provided, try to authenticate the token
# with the authentication providers that are associated with the collection. # with the authentication providers that are associated with the collection.
if not is_token_name: if not is_token_name:
@ -279,11 +271,13 @@ def get_token_store(
if not incoming: if not incoming:
raise HTTPException( raise HTTPException(
status_code=HTTP_401_UNAUTHORIZED, status_code=HTTP_401_UNAUTHORIZED,
detail='No incoming area for collection ' + collection_name detail='No incoming area for collection ' + collection_name,
) )
# Check whether a store for this collection and token does already exist. # Check whether a store for this collection and token does already exist.
store_info = instance_state.incoming_stores[collection_name].get(token_representation) store_info = instance_state.incoming_stores[collection_name].get(
token_representation
)
if store_info: if store_info:
return store_info return store_info
@ -304,11 +298,13 @@ def get_token_store(
def create_store( def create_store(
abstract_configuration: Configuration, abstract_configuration: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
) -> _ModelStore: ) -> _ModelStore:
collection_curated_path = abstract_configuration.collections[collection_name].curated collection_curated_path = abstract_configuration.collections[
collection_name
].curated
return create_token_store( return create_token_store(
abstract_configuration=abstract_configuration, abstract_configuration=abstract_configuration,
instance_state=instance_state, instance_state=instance_state,
@ -318,13 +314,13 @@ def create_store(
def create_token_store( def create_token_store(
abstract_configuration: Configuration, abstract_configuration: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
collection_name: str, collection_name: str,
store_dir: Path, store_dir: Path,
) -> _ModelStore: ) -> _ModelStore:
from dump_things_service.backends.schema_type_layer import SchemaTypeLayer
from dump_things_service.abstract_config import get_backend_and_extension from dump_things_service.abstract_config import get_backend_and_extension
from dump_things_service.backends.schema_type_layer import SchemaTypeLayer
from dump_things_service.exceptions import ConfigError from dump_things_service.exceptions import ConfigError
from dump_things_service.store.model_store import ModelStore from dump_things_service.store.model_store import ModelStore
@ -354,7 +350,6 @@ def create_token_store(
backend_config = abstract_configuration.collections[collection_name].backend backend_config = abstract_configuration.collections[collection_name].backend
backend_name, extension = get_backend_and_extension(backend_config.type) backend_name, extension = get_backend_and_extension(backend_config.type)
if backend_name == 'record_dir': if backend_name == 'record_dir':
backend = create_record_dir_token_store_backend( backend = create_record_dir_token_store_backend(
store_dir=store_dir, store_dir=store_dir,
order_by=instance_state.order_by, order_by=instance_state.order_by,
@ -376,7 +371,9 @@ def create_token_store(
if extension == 'stl': if extension == 'stl':
backend = SchemaTypeLayer(backend=backend, schema=schema_uri) backend = SchemaTypeLayer(backend=backend, schema=schema_uri)
submission_tags = abstract_configuration.collections[collection_name].submission_tags submission_tags = abstract_configuration.collections[
collection_name
].submission_tags
return ModelStore( return ModelStore(
schema=schema_uri, schema=schema_uri,
backend=backend, backend=backend,
@ -388,14 +385,14 @@ def create_token_store(
def create_record_dir_token_store_backend( def create_record_dir_token_store_backend(
store_dir: Path, store_dir: Path,
order_by: list[str], order_by: list[str],
schema_uri: str, schema_uri: str,
mapping_function: str, mapping_function: str,
suffix: str, suffix: str,
) -> _RecordDirStore: ) -> _RecordDirStore:
from dump_things_service.instance_state import record_dir_config_file_name
from dump_things_service.backends.record_dir import RecordDirStore from dump_things_service.backends.record_dir import RecordDirStore
from dump_things_service.instance_state import record_dir_config_file_name
# Write the configuration to the store, if it does not yet exist. # Write the configuration to the store, if it does not yet exist.
if not (store_dir / record_dir_config_file_name).exists(): if not (store_dir / record_dir_config_file_name).exists():
@ -416,15 +413,16 @@ def create_record_dir_token_store_backend(
def write_record_dir_config( def write_record_dir_config(
path: Path, path: Path,
mapping_function: str, mapping_function: str,
schema: str, schema: str,
): ):
from dump_things_service.instance_state import record_dir_config_file_name from dump_things_service.instance_state import record_dir_config_file_name
record_dir_config_file_path = path / record_dir_config_file_name record_dir_config_file_path = path / record_dir_config_file_name
if not record_dir_config_file_path.exists(): if not record_dir_config_file_path.exists():
record_dir_config_file_path.write_text(f"""# RecordDir Config record_dir_config_file_path.write_text(
f"""# RecordDir Config
type: records type: records
version: 1 version: 1
schema: {schema} schema: {schema}
@ -435,9 +433,9 @@ idfx: {mapping_function}
def create_sqlite_token_store_backend( def create_sqlite_token_store_backend(
store_dir: Path, store_dir: Path,
order_by: list[str], order_by: list[str],
) -> _SQLiteBackend: ) -> _SQLiteBackend:
from dump_things_service.backends.sqlite import SQLiteBackend from dump_things_service.backends.sqlite import SQLiteBackend
from dump_things_service.backends.sqlite import ( from dump_things_service.backends.sqlite import (
record_file_name as sqlite_record_file_name, record_file_name as sqlite_record_file_name,
@ -450,26 +448,22 @@ def create_sqlite_token_store_backend(
def check_bounds( def check_bounds(
length: int | None, length: int | None, max_length: int, collection: str, alternative_url: str
max_length: int,
collection: str,
alternative_url: str
): ):
if length > max_length: if length > max_length:
raise HTTPException( raise HTTPException(
status_code=HTTP_413_CONTENT_TOO_LARGE, status_code=HTTP_413_CONTENT_TOO_LARGE,
detail=f"Too many records found in collection '{collection}'. " detail=f"Too many records found in collection '{collection}'. "
f'Please use pagination (/{collection}{alternative_url}).', f'Please use pagination (/{collection}{alternative_url}).',
) )
async def process_token( async def process_token(
abstract_config: Configuration, abstract_config: Configuration,
instance_state: InstanceState, instance_state: InstanceState,
api_key: str | None, api_key: str | None,
collection: str, collection: str,
) -> tuple[TokenPermission, _ModelStore]: ) -> tuple[TokenPermission, _ModelStore]:
if api_key is None: if api_key is None:
collection_config = get_collection_config_by_name(abstract_config, collection) collection_config = get_collection_config_by_name(abstract_config, collection)
token_store, token_permissions, user_id = get_token_store( token_store, token_permissions, user_id = get_token_store(
@ -480,7 +474,7 @@ async def process_token(
is_token_name=True, is_token_name=True,
) )
else: else:
token_store, token_permissions, user_id = get_token_store( token_store, token_permissions, _user_id = get_token_store(
abstract_config, abstract_config,
instance_state, instance_state,
collection, collection,
@ -492,16 +486,15 @@ async def process_token(
) )
# Check for maintenance mode # Check for maintenance mode
if collection in instance_state.maintenance_mode: if collection in instance_state.maintenance_mode and not (
if not ( final_permissions.curated_read
final_permissions.curated_read and final_permissions.curated_write
and final_permissions.curated_write and final_permissions.zones_access
and final_permissions.zones_access ):
): raise HTTPException(
raise HTTPException( status_code=HTTP_503_SERVICE_UNAVAILABLE,
status_code=HTTP_503_SERVICE_UNAVAILABLE, detail=f"Collection '{collection}' is in maintenance mode",
detail=f"Collection '{collection}' is in maintenance mode", )
)
if not final_permissions.incoming_read and not final_permissions.curated_read: if not final_permissions.incoming_read and not final_permissions.curated_read:
raise HTTPException( raise HTTPException(
@ -512,32 +505,26 @@ async def process_token(
def get_required_incoming_labels( def get_required_incoming_labels(
abstract_config: Configuration, abstract_config: Configuration,
collection_name: str, collection_name: str,
) -> set[str]: ) -> set[str]:
return set( return {x[1] for x in get_required_incoming_info(abstract_config, collection_name)}
map(
lambda x: x[1],
get_required_incoming_info(abstract_config, collection_name),
)
)
def get_required_incoming_info( def get_required_incoming_info(
abstract_config: Configuration, abstract_config: Configuration,
collection_name: str, collection_name: str,
) -> set[tuple[str, str]]: ) -> set[tuple[str, str]]:
return { return {
(token_name, this_collection_info.incoming_label) (token_name, this_collection_info.incoming_label)
for token_name, token_info in abstract_config.tokens.items() for token_name, token_info in abstract_config.tokens.items()
for this_collection_name, this_collection_info in token_info.collections.items() for this_collection_name, this_collection_info in token_info.collections.items()
if this_collection_name == collection_name and mode_mapping[ if this_collection_name == collection_name
TokenModes(this_collection_info.mode) and mode_mapping[TokenModes(this_collection_info.mode)].incoming_write is True
].incoming_write is True
} }
def var_escape( def var_escape(
name: str, name: str,
) -> str: ) -> str:
return name.replace('_', '___').replace('-', '_0_') return name.replace('_', '___').replace('-', '_0_')

35
dump_things_service/validate.py
View file

@ -33,15 +33,14 @@ from dump_things_service.utils import (
def validate_record( def validate_record(
collection: str, collection: str,
data: BaseModel | str, data: BaseModel | str,
class_name: str, class_name: str,
model: Any, model: Any,
input_format: Format, input_format: Format,
_: bool, _: bool,
api_key: str | None = Depends(api_key_header_scheme), api_key: str | None = Depends(api_key_header_scheme),
) -> JSONResponse: ) -> JSONResponse:
instance_state = get_instance_state() instance_state = get_instance_state()
abstract_config = get_config() abstract_config = get_config()
@ -63,7 +62,7 @@ def validate_record(
else api_key else api_key
) )
store, token_permissions, user_id = get_token_store( _store, token_permissions, _user_id = get_token_store(
abstract_config, abstract_config,
instance_state, instance_state,
collection, collection,
@ -82,18 +81,30 @@ def validate_record(
) )
if input_format == Format.ttl: if input_format == Format.ttl:
with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Conversion error'): with wrap_http_exception(
ValueError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Conversion error',
):
json_object = FormatConverter( json_object = FormatConverter(
abstract_config.collections[collection].schema_location, abstract_config.collections[collection].schema_location,
input_format=Format.ttl, input_format=Format.ttl,
output_format=Format.json, output_format=Format.json,
).convert(data, class_name) ).convert(data, class_name)
with wrap_http_exception(ValidationError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): with wrap_http_exception(
ValidationError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Validation error',
):
TypeAdapter(getattr(model, class_name)).validate_python(json_object) TypeAdapter(getattr(model, class_name)).validate_python(json_object)
else: else:
# Try to convert it into TTL to detect potential errors before storing # Try to convert it into TTL to detect potential errors before storing
# the record # the record
with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): with wrap_http_exception(
ValueError,
status_code=HTTP_422_UNPROCESSABLE_CONTENT,
header='Validation error',
):
instance_state.validators[collection].validate(data) instance_state.validators[collection].validate(data)
return JSONResponse(True) return JSONResponse(True)

65
pyproject.toml
View file

@ -1,5 +1,8 @@
[build-system] [build-system]
requires = ["hatchling"] requires = [
"hatchling",
"hatch-vcs",
]
build-backend = "hatchling.build" build-backend = "hatchling.build"
[project] [project]
@ -17,9 +20,7 @@ classifiers = [
"License :: OSI Approved :: MIT License", "License :: OSI Approved :: MIT License",
"Development Status :: 4 - Beta", "Development Status :: 4 - Beta",
"Programming Language :: Python", "Programming Language :: Python",
"Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.12",
"Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: CPython",
@ -42,9 +43,20 @@ dependencies = [
] ]
[project.urls] [project.urls]
Documentation = "https://hub.psychoinformatics.de/datalink/dump-things-server" Documentation = "https://hub.psychoinformatics.de/orinoco/dump-things-server"
Issues = "https://codeberg.org/datalink/dump-things-server/issues" Issues = "https://codeberg.org/datalink/dump-things-server/issues"
Source = "https://hub.psychoinformatics.de/datalink/dump-things-server" Source = "https://hub.psychoinformatics.de/orinoco/dump-things-server"
Changelog = "https://hub.psychoinformatics.de/orinoco/dump-things-server/src/branch/master/CHANGELOG.md"
[project.optional-dependencies]
# this is what readthedocs consumes to decide what needs to be installed
# for compiling the docs
docs = [
"pytest",
"sphinx",
"sphinx_rtd_theme",
"sphinx_autodoc_typehints",
]
[project.scripts] [project.scripts]
dump-things-service = "dump_things_service.main:main" dump-things-service = "dump_things_service.main:main"
@ -75,14 +87,36 @@ only-include = [
] ]
[tool.hatch.version] [tool.hatch.version]
path = "dump_things_service/__about__.py" source = "vcs"
[tool.hatch.build.hooks.vcs]
version-file = "dump_things_service/_version.py"
[tool.hatch.envs.types] [tool.hatch.envs.types]
extra-dependencies = [ extra-dependencies = [
"mypy>=1.0.0", "mypy>=1.0.0",
] ]
[tool.hatch.envs.types.scripts] [tool.hatch.envs.types.scripts]
check = "mypy --install-types --non-interactive {args:src tests}" check = "mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context {args:dump_things_service}"
[tool.hatch.envs.docs]
description = "build Sphinx-based docs"
# also see project.optional-dependencies.docs!
# this is not considered by readthedocs
extra-dependencies = [
"pytest",
"sphinx",
"sphinx_rtd_theme",
"sphinx-autodoc-typehints",
]
[tool.hatch.envs.docs.scripts]
build = [
"make -C docs html",
]
clean = [
"rm -rf docs/generated",
"make -C docs clean",
]
[tool.coverage.run] [tool.coverage.run]
source_pkgs = ["dump_things_service"] source_pkgs = ["dump_things_service"]
@ -106,21 +140,19 @@ description = "fastapi dev environment"
[tool.hatch.envs.fastapi.scripts] [tool.hatch.envs.fastapi.scripts]
run = "python -m dump_things_service.main {args}" run = "python -m dump_things_service.main {args}"
[[tool.hatch.envs.tests.matrix]] [[tool.hatch.envs.hatch-test.matrix]]
python = ["3.11", "3.12"] python = ["3.11", "3.12"]
[tool.hatch.envs.tests] [tool.hatch.envs.hatch-test]
default-args = ["dump_things_service"]
extra-dependencies = [ extra-dependencies = [
"freezegun", "freezegun",
"httpx", "httpx2",
"pytest", "pytest",
"pytest-cov", "pytest-cov",
"pytest-httpserver", "pytest-httpserver",
] ]
[tool.hatch.envs.tests.scripts]
run = 'python -m pytest {args}'
[tool.ruff] [tool.ruff]
extend-exclude = [ extend-exclude = [
# sphinx # sphinx
@ -130,7 +162,7 @@ extend-exclude = [
] ]
line-length = 88 line-length = 88
indent-width = 4 indent-width = 4
target-version = "py39" target-version = "py311"
[tool.ruff.format] [tool.ruff.format]
# Prefer single quotes over double quotes. # Prefer single quotes over double quotes.
quote-style = "single" quote-style = "single"
@ -152,3 +184,6 @@ skip = '.git*'
check-hidden = true check-hidden = true
# ignore-regex = '' # ignore-regex = ''
# ignore-words-list = '' # ignore-words-list = ''
[tool.mypy]
disable_error_code = ["import-untyped"]