diff --git a/.forgejo/workflows/codespell.yml b/.forgejo/workflows/codespell.yml index 93e4437..5b2ce9f 100644 --- a/.forgejo/workflows/codespell.yml +++ b/.forgejo/workflows/codespell.yml @@ -2,7 +2,7 @@ --- name: Codespell -on: workflow_dispatch +on: [push, pull_request, workflow_dispatch] permissions: contents: read @@ -10,13 +10,13 @@ permissions: jobs: codespell: name: Check for spelling errors - runs-on: ubuntu-latest + runs-on: debian-latest steps: - name: Checkout uses: actions/checkout@v5 - name: Codespell - uses: codespell-project/actions-codespell@v2 + uses: https://github.com/codespell-project/actions-codespell@v2 with: ignore_words_list: crate diff --git a/.forgejo/workflows/mypy-pr.yml b/.forgejo/workflows/mypy-pr.yml new file mode 100644 index 0000000..024c185 --- /dev/null +++ b/.forgejo/workflows/mypy-pr.yml @@ -0,0 +1,36 @@ +name: Type annotation (PR) + +on: + pull_request: + paths: + - 'dump_things_service/**.py' + - '!**/tests/**.py' + +jobs: + check: + runs-on: debian-latest + steps: + - name: Checkout project + uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install hatch + run: uv tool install hatch + - name: Get Python changed files + id: changed-py-files + uses: https://github.com/tj-actions/changed-files@v46 + with: + files: | + *.py + **/*.py + - name: Type check changed files + if: steps.changed-py-files.outputs.any_changed == 'true' + run: | + # get any type stubs that mypy thinks it needs + # run mypy on the modified files only, and do not even follow imports. + # this results is a fairly superficial test, but given the overall + # state of annotations, we strive to become more correct incrementally + # with focused error reports, rather than barfing a huge complaint + # that is unrelated to the changeset someone has been working on. + # run on the oldest supported Python version + hatch run types:mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context ${{ steps.changed-py-files.outputs.all_changed_files }} diff --git a/.forgejo/workflows/ruff.yml b/.forgejo/workflows/ruff.yml new file mode 100644 index 0000000..1b0ddc1 --- /dev/null +++ b/.forgejo/workflows/ruff.yml @@ -0,0 +1,17 @@ +name: Ruff +on: [push, pull_request] +jobs: + ruff: + name: Code linting + runs-on: debian-latest + steps: + - name: Checkout project + uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install hatch + run: uv tool install hatch + - name: Check code + run: hatch check code + - name: Check formatting + run: hatch check fmt diff --git a/.forgejo/workflows/run_tests.yaml b/.forgejo/workflows/run_tests.yaml index 6630ca2..03a2241 100644 --- a/.forgejo/workflows/run_tests.yaml +++ b/.forgejo/workflows/run_tests.yaml @@ -12,9 +12,6 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - - name: Install uv uses: astral-sh/setup-uv@v6 @@ -23,14 +20,14 @@ jobs: - name: Run tests run: | - hatch run tests:run \ + hatch test \ --ignore=dump_things_service/tests/test_generators.py \ --ignore=dump_things_service/tests/test_ifabsent_patch.py - name: Run generator tests run: | - hatch run tests:run dump_things_service/tests/test_generators.py + hatch test dump_things_service/tests/test_generators.py - name: Run ifabsent-patch tests run: | - hatch run tests:run dump_things_service/tests/test_ifabsent_patch.py + hatch test dump_things_service/tests/test_ifabsent_patch.py diff --git a/.gitignore b/.gitignore index fba313f..6351760 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ dist/** tmp/** **/__pycache__ **/.hypothesis +.*.swp +dump_things_service/_version.py diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..630c0c1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,31 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGELOG.md b/CHANGELOG.md index 60a3533..e78b17d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,7 +123,7 @@ 3. The top-level mapping `admin_tokens` was added. - Configuration files are no longer read when the service is started. Instead - the service reads its configuration from the store, if it is present. Thw tool + the service reads its configuration from the store, if it is present. The tool (`dump-things-load-config`) can read an existing configuration file and manifest the described configuration on a running dump-things server. It supports pre version 6 config files and converts them to the new diff --git a/README.md b/README.md index ec6cc93..e208c9c 100644 --- a/README.md +++ b/README.md @@ -594,7 +594,7 @@ Most endpoints require a *collection*. These correspond to the names of the "dat The service provides the following user endpoints (In addition to user endpoints, there exist endpoints for curators. To view them, check the `/docs`-path in an installed service): -- `POST /maintenance`: this endpoint allows to set a collection into mantenance mode. +- `POST /maintenance`: this endpoint allows to set a collection into maintenance mode. In maintenance mode, only tokens with curator-privileges can access the collection. The posted data is a JSON that contains the name of the collection and whether the maintenance state should be active or not, for example: ```json diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..9719ae4 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +_build +generated diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..973b097 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= --fail-on-warning +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..b79be80 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,48 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +import dump_things_service + +project = 'dump-things-server' +copyright = '2025-2026, Christian Mönch' +author = 'Christian Mönch' +release = dump_things_service.__version__ + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'sphinx.ext.autosummary', + 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', + 'sphinx.ext.viewcode', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +primary_domain = 'py' +autoclass_content = "both" + +typehints_use_signature = True +typehints_use_signature_return = True + +# we build some docstrings from loguru. define some no-op substitutions +# to avoid errors +rst_prolog = """ +.. |Logger| replace:: Logger +.. |add| replace:: add +.. |sys.stderr| replace:: sys.stderr +.. |str.format| replace:: str.format +""" + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..17022f1 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,10 @@ +The `dump-thing-server` documentation +===================================== + +HERE BE CONTENT... + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/dump_things_service/__init__.py b/dump_things_service/__init__.py index cdb6a1a..fd2090d 100644 --- a/dump_things_service/__init__.py +++ b/dump_things_service/__init__.py @@ -20,8 +20,9 @@ from starlette.status import ( HTTP_503_SERVICE_UNAVAILABLE, ) +from dump_things_service._version import __version__ + __all__ = [ - 'Format', 'HTTP_200_OK', 'HTTP_201_CREATED', 'HTTP_300_MULTIPLE_CHOICES', @@ -37,6 +38,8 @@ __all__ = [ 'HTTP_503_SERVICE_UNAVAILABLE', 'JSON', 'YAML', + 'Format', + '__version__', 'config_file_name', 'reserved_collection_names', ] diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 384d90c..cfca7f0 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -1,14 +1,13 @@ import enum import hashlib import logging +from collections.abc import Callable, Iterable from functools import partial from pathlib import ( Path, PurePosixPath, ) from typing import ( - Callable, - Iterable, Literal, cast, ) @@ -17,7 +16,8 @@ from fastapi import HTTPException from pydantic import ( BaseModel, ConfigDict, - Field, ValidationError, + Field, + ValidationError, ) from yaml.scanner import ScannerError @@ -27,12 +27,11 @@ from dump_things_service import ( ) from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.backends.record_dir import ( - _RecordDirStore, RecordDirStore, + _RecordDirStore, ) from dump_things_service.exceptions import ConfigError - logger = logging.getLogger('dump_things_service') g_abstract_configuration = None @@ -103,7 +102,9 @@ class CollectionConfig(BaseModel): curated: PurePosixPath schema_location: str = Field(alias='schema') incoming: PurePosixPath | None = None - backend: RecordDirBackendConfig | SQLiteBackendConfig = RecordDirBackendConfig(type='record_dir+stl') + backend: RecordDirBackendConfig | SQLiteBackendConfig = RecordDirBackendConfig( + type='record_dir+stl' + ) auth_sources: list[ForgejoAuthSpec | ConfigAuthSpec] = [ConfigAuthSpec()] audit_backends: list[GitAuditBackendConfig] = [] submission_tags: TagSpec = TagSpec() @@ -200,7 +201,7 @@ def get_token_permissions(mode: str) -> TokenPermission: def get_config_backends( - store_path: Path, + store_path: Path, ) -> tuple[_RecordDirStore, GitAuditBackend]: global config_audit global config_backend @@ -211,9 +212,7 @@ def get_config_backends( if config_backend is None: config_backend = RecordDirStore( - config_path, - mapping_functions[MappingMethod.digest_md5], - 'yaml' + config_path, mapping_functions[MappingMethod.digest_md5], 'yaml' ) audit_path = store_path / config_audit_path @@ -226,8 +225,8 @@ def get_config_backends( def read_config( - store_path: Path, - force_reload: bool = False, + store_path: Path, + force_reload: bool = False, ) -> Configuration: global g_abstract_configuration @@ -244,7 +243,7 @@ def read_config( if record_info else Configuration( type='collections', - version = 2, + version=2, ) ) except ValidationError as ve: @@ -259,12 +258,12 @@ def get_config() -> Configuration: if not g_abstract_configuration: msg = 'Configuration not yet loaded' raise RuntimeError(msg) - return cast(Configuration, g_abstract_configuration) + return cast('Configuration', g_abstract_configuration) def store_config( - store_path, - config: Configuration, + store_path, + config: Configuration, ): global g_abstract_configuration @@ -274,7 +273,7 @@ def store_config( config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object=json_object + json_object=json_object, ) audit_backend.add_record( record=json_object, @@ -284,8 +283,8 @@ def store_config( def tokens_for_collection( - config: Configuration, - collection: str, + config: Configuration, + collection: str, ) -> Iterable[TokenConfig]: yield from ( token @@ -295,8 +294,8 @@ def tokens_for_collection( def check_collection( - abstract_config: Configuration, - collection: str, + abstract_config: Configuration, + collection: str, ): if collection not in abstract_config.collections: raise HTTPException( @@ -306,18 +305,17 @@ def check_collection( def check_label( - store_path: Path, - abstract_config: Configuration, - collection: str, - label: str, + store_path: Path, + abstract_config: Configuration, + collection: str, + label: str, ): from dump_things_service.utils import get_on_disk_labels """Check that a label exists in a collection configuration or on disk""" - if ( - label not in get_config_labels(abstract_config, collection) - and label not in get_on_disk_labels(store_path, abstract_config, collection) - ): + if label not in get_config_labels( + abstract_config, collection + ) and label not in get_on_disk_labels(store_path, abstract_config, collection): raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"No incoming label: '{label}' in collection: '{collection}'.", @@ -325,8 +323,8 @@ def check_label( def get_config_labels( - abstract_config: Configuration, - collection: str, + abstract_config: Configuration, + collection: str, ) -> set[str]: check_collection(abstract_config, collection) return { @@ -336,17 +334,14 @@ def get_config_labels( } -def get_default_token_name( - abstract_config: Configuration, - collection: str -) -> str: +def get_default_token_name(abstract_config: Configuration, collection: str) -> str: check_collection(abstract_config, collection) return abstract_config.collections[collection].default_token def get_token_info_by_representation( - abstract_config: Configuration, - token_representation: str, + abstract_config: Configuration, + token_representation: str, ) -> tuple[str, TokenConfig] | None: """Get the name of the token given in `token_representation`""" hashed_representation = hash_token_representation(token_representation) @@ -361,23 +356,22 @@ def get_token_info_by_representation( def hash_token_representation( - token_representation: str, + token_representation: str, ) -> str: return hashlib.sha256(token_representation.encode()).hexdigest() def get_token_config_by_name( - abstract_config: Configuration, - token_name: str, + abstract_config: Configuration, + token_name: str, ) -> TokenConfig | None: return abstract_config.tokens.get(token_name) def get_token_infos_for_collection( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> Iterable[tuple[str, TokenConfig, TokenCollectionConfig]]: - yield from { (token_name, token_config, token_collection_config) for token_name, token_config in abstract_config.tokens.items() @@ -387,11 +381,10 @@ def get_token_infos_for_collection( def get_token_config_for_representation_and_collection( - abstract_config: Configuration, - collection_name: str, - token_representation: str, + abstract_config: Configuration, + collection_name: str, + token_representation: str, ) -> tuple[str, TokenConfig, TokenCollectionConfig] | None: - token_info = get_token_info_by_representation( abstract_config=abstract_config, token_representation=token_representation, @@ -405,8 +398,8 @@ def get_token_config_for_representation_and_collection( def get_collection_config_by_name( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> CollectionConfig: collection_config = abstract_config.collections.get(collection_name) if not collection_config: @@ -418,10 +411,9 @@ def get_collection_config_by_name( def get_default_token_config( - abstract_config: Configuration, - collection: str, + abstract_config: Configuration, + collection: str, ) -> TokenConfig | None: - default_token_name = get_collection_config_by_name( abstract_config, collection, @@ -445,18 +437,18 @@ def get_hex_digest(hasher: Callable, data: str) -> str: def mapping_digest_p3( - hasher: Callable, - pid: str, - suffix: str, + hasher: Callable, + pid: str, + suffix: str, ) -> Path: hex_digest = get_hex_digest(hasher, pid) return Path(hex_digest[:3]) / (hex_digest[3:] + '.' + suffix) def mapping_digest_p3_p3( - hasher: Callable, - pid: str, - suffix: str, + hasher: Callable, + pid: str, + suffix: str, ) -> Path: hex_digest = get_hex_digest(hasher, pid) return Path(hex_digest[:3]) / hex_digest[3:6] / (hex_digest[6:] + '.' + suffix) diff --git a/dump_things_service/admin.py b/dump_things_service/admin.py index 20a19c0..7f94ee8 100644 --- a/dump_things_service/admin.py +++ b/dump_things_service/admin.py @@ -9,14 +9,13 @@ from dump_things_service.abstract_config import ( ) from dump_things_service.instance_state import InstanceState - logger = logging.getLogger('dump_things_service') def authenticate_admin( - instance_state: InstanceState, - abstract_config: Configuration, - api_key: str, + instance_state: InstanceState, + abstract_config: Configuration, + api_key: str, ): if api_key: hashed_token_representation = hash_token_representation(api_key) diff --git a/dump_things_service/audit/__init__.py b/dump_things_service/audit/__init__.py index 58b8458..68f671f 100644 --- a/dump_things_service/audit/__init__.py +++ b/dump_things_service/audit/__init__.py @@ -7,10 +7,10 @@ from abc import ( class AuditBackend(metaclass=ABCMeta): @abstractmethod def add_record( - self, - record: dict, - committer_id: str, - author_id: str | None = None, + self, + record: dict, + committer_id: str, + author_id: str | None = None, ) -> None: """Add information about a new record version to the audit log @@ -35,8 +35,8 @@ class AuditBackend(metaclass=ABCMeta): @abstractmethod def get_audit_log( - self, - record_id: str, + self, + record_id: str, ) -> dict: """Get the content of the audit log diff --git a/dump_things_service/audit/gitaudit.py b/dump_things_service/audit/gitaudit.py index b3afaed..a3b3289 100644 --- a/dump_things_service/audit/gitaudit.py +++ b/dump_things_service/audit/gitaudit.py @@ -6,6 +6,7 @@ committed. Changes are annotated with a time stamp and a user-id """ + from __future__ import annotations import hashlib @@ -23,21 +24,20 @@ import yaml from datalad_core.git_utils import apply_changeset from datalad_core.repo import Repo from datalad_core.runners import ( - call_git, CommandError, + call_git, ) -from . import AuditBackend - +from dump_things_service.audit import AuditBackend index_file_name = 'gitaudit_index.log' class FlushingThread(Thread): def __init__( - self, - backend: GitAuditBackend, - auto_flush_timeout: int, + self, + backend: GitAuditBackend, + auto_flush_timeout: int, ): super().__init__() self.auto_flush_timeout = auto_flush_timeout @@ -56,11 +56,10 @@ class FlushingThread(Thread): class GitAuditBackend(AuditBackend): - def __init__( - self, - path: Path, - auto_flush_timeout: int = 60, + self, + path: Path, + auto_flush_timeout: int = 60, ): self.path = path self.index_path = None @@ -69,7 +68,8 @@ class GitAuditBackend(AuditBackend): self.lock = Lock() self.last_flush_time = 0 if auto_flush_timeout < 1: - raise ValueError('auto_flush_timeout must be greater or equal to 1') + msg = 'auto_flush_timeout must be greater or equal to 1' + raise ValueError(msg) self.flushing_thread = FlushingThread(self, auto_flush_timeout) self.flushing_thread.start() self._init_repo() @@ -82,10 +82,10 @@ class GitAuditBackend(AuditBackend): self.flushing_thread = None def add_record( - self, - record: dict, - committer_id: str, - author_id: str | None = None, + self, + record: dict, + committer_id: str, + author_id: str | None = None, ) -> None: with self.lock: author_id = committer_id if author_id is None else author_id @@ -109,15 +109,15 @@ class GitAuditBackend(AuditBackend): self.last_flush_time = time.time() def get_audit_log( - self, - record_id: str, + self, + record_id: str, ) -> dict: with self.lock: return self._locked_get_audit_log(record_id) def _locked_get_audit_log( - self, - record_id: str, + self, + record_id: str, ) -> dict: self._locked_flush() @@ -125,32 +125,42 @@ class GitAuditBackend(AuditBackend): # the records changes = [] yaml_location, log_location = map(str, self._get_location_for(record_id)[1:]) - commit_hashes = call_git( - ['log', '--format=%H', '--', log_location], - cwd=self.path, - capture_output=True, - ).decode().splitlines() - for commit_hash in commit_hashes: - log_diff_lines = call_git( - ['show', '--format=%b', commit_hash, '--', log_location], + commit_hashes = ( + call_git( + ['log', '--format=%H', '--', log_location], cwd=self.path, capture_output=True, - ).decode().splitlines() + ) + .decode() + .splitlines() + ) + for commit_hash in commit_hashes: + log_diff_lines = ( + call_git( + ['show', '--format=%b', commit_hash, '--', log_location], + cwd=self.path, + capture_output=True, + ) + .decode() + .splitlines() + ) # Get the log entry - log_line = tuple( - filter( + log_line = next(filter( lambda l: not l.startswith('+++') and l.startswith('+'), log_diff_lines, - ) - )[0][1:] + ))[1:] log_entry = json.loads(log_line) # Get the YAML diff - yaml_diff_lines = call_git( - ['show', '--format=%b', commit_hash, '--', yaml_location], - cwd=self.path, - capture_output=True, - ).decode().splitlines() + yaml_diff_lines = ( + call_git( + ['show', '--format=%b', commit_hash, '--', yaml_location], + cwd=self.path, + capture_output=True, + ) + .decode() + .splitlines() + ) yaml_diff = '\n'.join(filter(lambda l: l != '', yaml_diff_lines)) + '\n' # Get the YAML content @@ -173,15 +183,15 @@ class GitAuditBackend(AuditBackend): return {c[0]: c[1:] for c in changes} def get_audit_logs( - self, - record_id_pattern: str, + self, + record_id_pattern: str, ) -> dict: with self.lock: return self._locked_get_audit_logs(record_id_pattern) def _locked_get_audit_logs( - self, - record_id_pattern: str, + self, + record_id_pattern: str, ) -> dict: self._locked_flush() matcher = re.compile(record_id_pattern) @@ -197,12 +207,12 @@ class GitAuditBackend(AuditBackend): } def _add_elements( - self, - record_id: str, - location: tuple[str, Path, Path], - committer_id: str, - author_id: str, - record: dict, + self, + record_id: str, + location: tuple[str, Path, Path], + committer_id: str, + author_id: str, + record: dict, ) -> bool: existing_record = self._read_record_from_repo_path(location[1]) if existing_record != record: @@ -218,10 +228,10 @@ class GitAuditBackend(AuditBackend): return False def _add_log_entry( - self, - log_location: Path, - committer_id: str, - author_id: str, + self, + log_location: Path, + committer_id: str, + author_id: str, ) -> None: time_stamp = datetime.now().isoformat() entry = { @@ -234,20 +244,20 @@ class GitAuditBackend(AuditBackend): self.current_change_set[log_location] = log_content def _add_index_entry( - self, - record_id: str, + self, + record_id: str, ): if record_id not in self.index: self.cached_index_entries.append(record_id) self.index.add(record_id) def _read_from_repo_path( - self, - path: Path, + self, + path: Path, ) -> bytes: try: return call_git( - ['cat-file', '-p', f'master:{str(path)}'], + ['cat-file', '-p', f'master:{path!s}'], cwd=self.path, capture_output=True, ) @@ -257,14 +267,14 @@ class GitAuditBackend(AuditBackend): raise def _read_record_from_repo_path( - self, - path: Path, + self, + path: Path, ): return yaml.safe_load(self._read_from_repo_path(path)) def _has_pending_changes( - self, - location: tuple[str, Path, Path], + self, + location: tuple[str, Path, Path], ) -> bool: log_pending = location[1] in self.current_change_set record_pending = location[2] in self.current_change_set @@ -286,11 +296,11 @@ class GitAuditBackend(AuditBackend): self.current_change_set = {} def _get_location_for( - self, - record_id: str, + self, + record_id: str, ) -> tuple[str, Path, Path]: base = hashlib.sha1(record_id.encode()).hexdigest() - dir_1, dir_2, name = base[0:3], base[3:6], base[6:] + dir_1, dir_2, _name = base[0:3], base[3:6], base[6:] location_dir = Path(dir_1) / Path(dir_2) return ( base, @@ -321,28 +331,32 @@ class GitAuditBackend(AuditBackend): if not self.index_path.exists(): self._rebuild_index() - with open(self.index_path, 'rt') as f: - self.index = set(line.strip() for line in f.readlines()) + with open(self.index_path) as f: + self.index = {line.strip() for line in f} def _add_to_index( - self, - record_id: str, + self, + record_id: str, ): if record_id not in self.index: self.cached_index_entries.append(record_id) self.index.add(record_id) def _rebuild_index(self): - tree_entries = call_git( - ['ls-tree', '-r', 'master:'], - cwd=self.path, - capture_output=True, - ).decode().splitlines() - with open(self.index_path, 'wt') as f: + tree_entries = ( + call_git( + ['ls-tree', '-r', 'master:'], + cwd=self.path, + capture_output=True, + ) + .decode() + .splitlines() + ) + with open(self.index_path, 'w') as f: for line in tree_entries: if not line.endswith('.yaml'): continue - flag, object_type, object_hash, file_name = line.split(maxsplit=3) + _flag, _object_type, object_hash, _file_name = line.split(maxsplit=3) record = yaml.safe_load( call_git( ['show', object_hash], diff --git a/dump_things_service/audit/tests/test_gitaudit.py b/dump_things_service/audit/tests/test_gitaudit.py index ef1e059..adb93b5 100644 --- a/dump_things_service/audit/tests/test_gitaudit.py +++ b/dump_things_service/audit/tests/test_gitaudit.py @@ -19,7 +19,7 @@ def _get_audit_log_lines(backend: GitAuditBackend, record_id: str) -> list[str]: def test_gitaudit_basic(tmp_path_factory): - tmp_path = tmp_path_factory.mktemp("gitaudit_backend") + tmp_path = tmp_path_factory.mktemp('gitaudit_backend') backend = GitAuditBackend(tmp_path) @@ -44,14 +44,13 @@ def test_gitaudit_basic(tmp_path_factory): # Check that the changes are reported changes = backend.get_audit_log(record_id) assert len(changes) == 4 - assert tuple(map(lambda e: e[0:2], changes.values())) == tuple( - (f'committer_{100 + i}@x.org', f'author_{i}@y.org') - for i in range(4) + assert tuple(e[0:2] for e in changes.values()) == tuple( + (f'committer_{100 + i}@x.org', f'author_{i}@y.org') for i in range(4) ) def test_gitaudit_identical_change(tmp_path_factory): - tmp_path = tmp_path_factory.mktemp("gitaudit_backend") + tmp_path = tmp_path_factory.mktemp('gitaudit_backend') backend = GitAuditBackend(tmp_path) @@ -59,13 +58,13 @@ def test_gitaudit_identical_change(tmp_path_factory): backend.add_record( record={'pid': record_id}, committer_id='committer_b@x.org', - author_id = 'author_b@y.org', + author_id='author_b@y.org', ) backend.add_record( record={'pid': record_id}, committer_id='committer_b@x.org', - author_id = 'author_b@y.org', + author_id='author_b@y.org', ) # Check that there is only one entry in the audit log @@ -83,7 +82,7 @@ def test_gitaudit_identical_change(tmp_path_factory): def test_gitaudit_huge_log(tmp_path_factory): - tmp_path = tmp_path_factory.mktemp("gitaudit_backend") + tmp_path = tmp_path_factory.mktemp('gitaudit_backend') backend = GitAuditBackend(tmp_path) @@ -96,7 +95,7 @@ def test_gitaudit_huge_log(tmp_path_factory): backend.add_record( record={'pid': record_id, 'content': f'j:{j}, i:{i}'}, committer_id='committer@x.org', - author_id = 'author@y.org', + author_id='author@y.org', ) # Check that the changes are reported diff --git a/dump_things_service/auth/__init__.py b/dump_things_service/auth/__init__.py index 051720f..00ff138 100644 --- a/dump_things_service/auth/__init__.py +++ b/dump_things_service/auth/__init__.py @@ -8,6 +8,7 @@ determine: - the incoming_label to be used with the token """ + from __future__ import annotations import abc diff --git a/dump_things_service/auth/config.py b/dump_things_service/auth/config.py index 8f8976e..1808214 100644 --- a/dump_things_service/auth/config.py +++ b/dump_things_service/auth/config.py @@ -1,31 +1,30 @@ -"""Use configuration information to fetch token permissions, ids, and incoming_label """ +"""Use configuration information to fetch token permissions, ids, and incoming_label""" -from dump_things_service.abstract_config import Configuration +from dump_things_service.abstract_config import ( + Configuration, + get_token_config_for_representation_and_collection, + get_token_permissions, +) from dump_things_service.auth import ( AuthenticationInfo, AuthenticationSource, InvalidTokenError, ) -from dump_things_service.abstract_config import ( - get_token_permissions, - get_token_config_for_representation_and_collection, -) class ConfigAuthenticationSource(AuthenticationSource): def __init__( - self, - abstract_configuration: Configuration, - collection_name: str, + self, + abstract_configuration: Configuration, + collection_name: str, ): self.abstract_configuration = abstract_configuration self.collection_name = collection_name def authenticate( - self, - token_representation: str, + self, + token_representation: str, ) -> AuthenticationInfo: - result = get_token_config_for_representation_and_collection( self.abstract_configuration, self.collection_name, diff --git a/dump_things_service/auth/forgejo.py b/dump_things_service/auth/forgejo.py index 3b99524..72c018e 100644 --- a/dump_things_service/auth/forgejo.py +++ b/dump_things_service/auth/forgejo.py @@ -7,13 +7,14 @@ Note: for some reason, the request: does not require a token. If the owner and the repo are known, the request will emit a complete repository-record including the complete owner-record. """ + from __future__ import annotations import hashlib import logging import time from functools import wraps -from typing import Callable +from typing import TYPE_CHECKING import requests from requests.exceptions import Timeout @@ -22,13 +23,16 @@ from dump_things_service import ( HTTP_300_MULTIPLE_CHOICES, HTTP_401_UNAUTHORIZED, ) +from dump_things_service.abstract_config import TokenPermission from dump_things_service.auth import ( AuthenticationError, AuthenticationInfo, AuthenticationSource, InvalidTokenError, ) -from dump_things_service.abstract_config import TokenPermission + +if TYPE_CHECKING: + from collections.abc import Callable logger = logging.getLogger('dump_things_service') @@ -46,7 +50,8 @@ class MethodCache: def cache_temporary( duration: int = 300, ) -> Callable: - """ Cache results for a given time (default: 300 seconds) """ + """Cache results for a given time (default: 300 seconds)""" + def decorator(func: Callable) -> Callable: @wraps(func) def wrapper(*args, **kwargs): @@ -56,12 +61,15 @@ class MethodCache: if cached_data is None or time.time() - cached_data[0] > duration: self.__cached_data[key] = (time.time(), func(*args, **kwargs)) return self.__cached_data[key][1] + return wrapper + return decorator class RemoteAuthenticationError(AuthenticationError): """Exception for remote authentication errors.""" + def __init__(self, status: int, message: str): self.status = status self.message = message @@ -133,14 +141,15 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): ) from e if r.status_code >= HTTP_300_MULTIPLE_CHOICES: - msg = f'invalid token: ({r.status_code}): {r.text}' + cleaned_text = r.text.replace(token, '***') + msg = f'invalid token: ({r.status_code}): {cleaned_text}' raise InvalidTokenError(msg) return r.json() @MethodCache.cache_temporary(duration=120) def _get_user( - self, - token: str, + self, + token: str, ) -> dict: return self._get_json_from_endpoint('user', token) @@ -183,8 +192,8 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): @staticmethod def _get_permissions( - code_permission: str, - action_permission: str, + code_permission: str, + action_permission: str, ) -> TokenPermission: is_curator = action_permission == 'write' read = code_permission in ('read', 'write') or is_curator @@ -197,11 +206,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): zones_access=is_curator, ) - def _get_unit_content( - self, - team: dict, - unit_name: str - ) -> str: + def _get_unit_content(self, team: dict, unit_name: str) -> str: permissions = team['units_map'].get(unit_name) if not permissions: logger.debug(f'no unit `repo.actions` in team {self.team}') @@ -216,23 +221,22 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): return permissions def _instance_label(self) -> str: - return self.instance_id or hashlib.md5( - self.api_url.encode() - ).hexdigest() + return self.instance_id or hashlib.md5(self.api_url.encode()).hexdigest() @MethodCache.cache_temporary(duration=60) def authenticate( self, token: str, ) -> AuthenticationInfo: - - logger.debug(f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}') + logger.debug( + f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}' + ) user_teams = self._get_teams_for_user(token) logger.debug(f'user_teams: {user_teams}') if self.team not in user_teams: - logger.debug(f'{self.team} not in user\'s teams') + logger.debug(f"{self.team} not in user's teams") msg = f'token user is not member of team `{self.team}`' raise RemoteAuthenticationError( status=HTTP_401_UNAUTHORIZED, @@ -281,8 +285,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): action_permissions, ), user_id=user_info['email'], - incoming_label= - f'forgejo-{self._instance_label()}-team-{organization["name"]}-{team["name"]}' - if self.label_type == 'team' - else f'forgejo-{self._instance_label()}-user-{user_info["login"]}', + incoming_label=f'forgejo-{self._instance_label()}-team-{organization["name"]}-{team["name"]}' + if self.label_type == 'team' + else f'forgejo-{self._instance_label()}-user-{user_info["login"]}', ) diff --git a/dump_things_service/authenticate.py b/dump_things_service/authenticate.py index cf4793b..f45388c 100644 --- a/dump_things_service/authenticate.py +++ b/dump_things_service/authenticate.py @@ -1,56 +1,34 @@ from __future__ import annotations -import logging -from itertools import count from typing import TYPE_CHECKING from fastapi import ( - APIRouter, - Depends, - FastAPI, HTTPException, ) -from fastapi_pagination import ( - Page, - add_pagination, - paginate, -) from dump_things_service import ( HTTP_401_UNAUTHORIZED, - HTTP_404_NOT_FOUND, - HTTP_422_UNPROCESSABLE_CONTENT, abstract_config, ) from dump_things_service.abstract_config import ( check_collection, read_config, ) -from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer -from dump_things_service.exceptions import CurieResolutionError from dump_things_service.instance_state import get_instance_state -from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, - check_bounds, - cleaned_json, - wrap_http_exception, ) if TYPE_CHECKING: - from pydantic import BaseModel - + from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends import StorageBackend - from dump_things_service.lazy_list import LazyList from dump_things_service.store.model_store import _ModelStore def get_store_and_backend( - collection: str, - plain_token: str | None, + collection: str, + plain_token: str | None, ) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]: - # A token is required if plain_token is None: raise HTTPException( diff --git a/dump_things_service/backends/__init__.py b/dump_things_service/backends/__init__.py index 901b9e8..74128cb 100644 --- a/dump_things_service/backends/__init__.py +++ b/dump_things_service/backends/__init__.py @@ -83,12 +83,12 @@ class BackendResultList(LazyList): @abstractmethod def generate_result( - self, - index: int, - iri: str, - class_name: str, - sort_key: str, - private: Any, + self, + index: int, + iri: str, + class_name: str, + sort_key: str, + private: Any, ) -> RecordInfo: """ Generate a record info object from the provided parameters. @@ -105,23 +105,21 @@ class BackendResultList(LazyList): class StorageBackend(metaclass=ABCMeta): def __init__( - self, - order_by: Iterable[str] | None = None, + self, + order_by: Iterable[str] | None = None, ): self.order_by = order_by or ['pid'] @abstractmethod - def get_uri( - self - ) -> str: + def get_uri(self) -> str: raise NotImplementedError @abstractmethod def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): raise NotImplementedError @@ -139,37 +137,37 @@ class StorageBackend(metaclass=ABCMeta): @abstractmethod def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: raise NotImplementedError @abstractmethod def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: raise NotImplementedError @abstractmethod def get_records_of_classes( - self, - class_names: Iterable[str], - pattern: str | None = None, + self, + class_names: Iterable[str], + pattern: str | None = None, ) -> BackendResultList: raise NotImplementedError @abstractmethod def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> BackendResultList: raise NotImplementedError def create_sort_key( - json_object: dict[str, Any], - order_by: Iterable[str], + json_object: dict[str, Any], + order_by: Iterable[str], ) -> str: return '-'.join( str(json_object.get(key)) if json_object.get(key) is not None else chr(0x10FFFF) diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 16e0258..40ef6c1 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -10,7 +10,6 @@ import logging from pathlib import Path from typing import ( TYPE_CHECKING, - Callable, ) import yaml @@ -26,12 +25,12 @@ from dump_things_service.backends import ( from dump_things_service.backends.record_dir_index import RecordDirIndex if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable __all__ = [ - '_RecordDirStore', 'RecordDirStore', + '_RecordDirStore', ] ignored_files = {'.', '..', config_file_name} @@ -45,12 +44,12 @@ class RecordDirResultList(BackendResultList): """ def generate_result( - self, - _: int, - iri: str, - class_name: str, - sort_key: str, - path: Path, + self, + _: int, + iri: str, + class_name: str, + sort_key: str, + path: Path, ) -> RecordInfo: """ Generate a JSON representation of the record at index `index`. @@ -76,11 +75,11 @@ class _RecordDirStore(StorageBackend): """Store records in a directory structure""" def __init__( - self, - root: Path, - pid_mapping_function: Callable, - suffix: str, - order_by: Iterable[str] | None = None, + self, + root: Path, + pid_mapping_function: Callable, + suffix: str, + order_by: Iterable[str] | None = None, ): super().__init__(order_by=order_by) if not root.is_absolute(): @@ -91,28 +90,26 @@ class _RecordDirStore(StorageBackend): self.suffix = suffix self.index = RecordDirIndex(root, suffix) - def get_uri( - self - ) -> str: + def get_uri(self) -> str: return f'file://{self.root!s}' def build_index( - self, - schema: str, + self, + schema: str, ): self.index.rebuild_index(schema, self.order_by) def build_index_if_needed( - self, - schema: str, + self, + schema: str, ): self.index.rebuild_if_needed(schema, self.order_by) def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): pid = json_object['pid'] @@ -148,8 +145,8 @@ class _RecordDirStore(StorageBackend): self.index.add_iri_info(iri, class_name, str(storage_path), sort_string) def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: index_entry = self.index.get_info_for_iri(iri) if index_entry is None: @@ -165,9 +162,9 @@ class _RecordDirStore(StorageBackend): ) def get_records_of_classes( - self, - class_names: list[str], - pattern: str | None = None, + self, + class_names: list[str], + pattern: str | None = None, ) -> RecordDirResultList: return RecordDirResultList().add_info( sorted( @@ -186,8 +183,8 @@ class _RecordDirStore(StorageBackend): ) def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> RecordDirResultList: return RecordDirResultList().add_info( sorted( @@ -205,8 +202,8 @@ class _RecordDirStore(StorageBackend): ) def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: index_entry = self.index.get_info_for_iri(iri) if index_entry is None: @@ -226,10 +223,10 @@ _existing_stores = {} def RecordDirStore( # noqa: N802 - root: Path, - pid_mapping_function: Callable, - suffix: str, - order_by: Iterable[str] | None = None, + root: Path, + pid_mapping_function: Callable, + suffix: str, + order_by: Iterable[str] | None = None, ) -> _RecordDirStore: """Get a record directory store for the given root directory.""" existing_store = _existing_stores.get(root) diff --git a/dump_things_service/backends/record_dir_index.py b/dump_things_service/backends/record_dir_index.py index d872dd6..510d49d 100644 --- a/dump_things_service/backends/record_dir_index.py +++ b/dump_things_service/backends/record_dir_index.py @@ -65,11 +65,11 @@ class IndexEntry(Base): class RecordDirIndex: def __init__( - self, - store_dir: Path, - suffix: str, - *, - echo: bool = False, + self, + store_dir: Path, + suffix: str, + *, + echo: bool = False, ): if not store_dir.is_absolute(): msg = f'Not an absolute path: {store_dir}' @@ -91,11 +91,11 @@ class RecordDirIndex: Base.metadata.create_all(self.engine) def add_iri_info( - self, - iri: str, - class_name: str, - path: str, - sort_key: str, + self, + iri: str, + class_name: str, + path: str, + sort_key: str, ): with Session(self.engine) as session, session.begin(): self.add_iri_info_with_session( @@ -107,12 +107,12 @@ class RecordDirIndex: ) def add_iri_info_with_session( - self, - session: Session, - iri: str, - class_name: str, - path: str, - sort_key: str, + self, + session: Session, + iri: str, + class_name: str, + path: str, + sort_key: str, ): existing_record = session.query(IndexEntry).filter_by(iri=iri).first() if existing_record: @@ -131,8 +131,8 @@ class RecordDirIndex: ) def get_info_for_iri( - self, - iri: str, + self, + iri: str, ) -> tuple | None: with Session(self.engine) as session, session.begin(): statement = select(IndexEntry).filter_by(iri=iri) @@ -142,8 +142,8 @@ class RecordDirIndex: return None def get_info_for_class( - self, - class_name: str, + self, + class_name: str, ) -> Generator[IndexEntry]: with Session(self.engine) as session, session.begin(): statement = select(IndexEntry).filter_by(class_name=class_name) @@ -152,7 +152,7 @@ class RecordDirIndex: yield row[0] def get_info_for_all_classes( - self, + self, ) -> Generator[IndexEntry]: statement = select(IndexEntry) with Session(self.engine) as session, session.begin(): @@ -161,8 +161,8 @@ class RecordDirIndex: yield row[0] def remove_iri_info( - self, - iri: str, + self, + iri: str, ) -> bool: statement = delete(IndexEntry).where(IndexEntry.iri == iri) with Session(self.engine) as session, session.begin(): @@ -170,9 +170,9 @@ class RecordDirIndex: return result.rowcount == 1 def rebuild_index( - self, - schema: str, - order_by: Iterable[str] | None = None, + self, + schema: str, + order_by: Iterable[str] | None = None, ): """Rebuild the index from the records in the directory.""" lgr.info('Building IRI index for records in %s', self.store_dir) @@ -223,17 +223,17 @@ class RecordDirIndex: self.needs_rebuild = False def rebuild_if_needed( - self, - schema: str, - order_by: Iterable[str] | None = None, + self, + schema: str, + order_by: Iterable[str] | None = None, ): if self.needs_rebuild: self.rebuild_index(schema=schema, order_by=order_by) self.needs_rebuild = False def _get_class_name( - self, - path: Path, + self, + path: Path, ) -> str: """Get the class name from the path.""" rel_path = path.absolute().relative_to(self.store_dir) diff --git a/dump_things_service/backends/schema_type_layer.py b/dump_things_service/backends/schema_type_layer.py index 640519a..6ce3594 100644 --- a/dump_things_service/backends/schema_type_layer.py +++ b/dump_things_service/backends/schema_type_layer.py @@ -34,16 +34,16 @@ if TYPE_CHECKING: __all__ = [ - '_SchemaTypeLayer', 'SchemaTypeLayer', + '_SchemaTypeLayer', ] class SchemaTypeLayerResultList(BackendResultList): def __init__( - self, - origin_list: BackendResultList, - schema_model: ModuleType, + self, + origin_list: BackendResultList, + schema_model: ModuleType, ): super().__init__() self.schema_model = schema_model @@ -51,12 +51,12 @@ class SchemaTypeLayerResultList(BackendResultList): self.list_info = self.origin_list.list_info def generate_result( - self, - index: int, - iri: str, - class_name: str, - sort_key: str, - private: Any, + self, + index: int, + iri: str, + class_name: str, + sort_key: str, + private: Any, ) -> RecordInfo: origin_element = self.origin_list.generate_result( index, iri, class_name, sort_key, private @@ -73,31 +73,28 @@ class _SchemaTypeLayer(StorageBackend): """Proxy backend that removes `schema_type` from stored records""" def __init__( - self, - backend: StorageBackend, - schema: str, + self, + backend: StorageBackend, + schema: str, ): super().__init__() self.backend = backend self.schema_model = get_schema_model_for_schema(schema) - def get_uri( - self - ) -> str: + def get_uri(self) -> str: return self.backend.get_uri() def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): # Remove the top level `schema_type` from the JSON object because we # don't want to store it in the files. We add `schema_type` after # reading the record from disk. The value of `schema_type` is determined # by the class name of the record, which is stored in the path. - if 'schema_type' in json_object: - del json_object['schema_type'] + json_object.pop('schema_type', None) self.backend.add_record( iri=iri, class_name=class_name, @@ -105,14 +102,14 @@ class _SchemaTypeLayer(StorageBackend): ) def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: return self.backend.remove_record(iri=iri) def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: origin_result = self.backend.get_record_by_iri(iri) if origin_result and 'schema_type' not in origin_result.json_object: @@ -123,9 +120,9 @@ class _SchemaTypeLayer(StorageBackend): return origin_result def get_records_of_classes( - self, - class_names: list[str], - pattern: str | None = None, + self, + class_names: list[str], + pattern: str | None = None, ) -> BackendResultList: return SchemaTypeLayerResultList( origin_list=self.backend.get_records_of_classes( @@ -136,8 +133,8 @@ class _SchemaTypeLayer(StorageBackend): ) def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> BackendResultList: return SchemaTypeLayerResultList( origin_list=self.backend.get_all_records(pattern), @@ -150,8 +147,8 @@ class _SchemaTypeLayer(StorageBackend): def _get_schema_type( - class_name: str, - schema_module: ModuleType, + class_name: str, + schema_module: ModuleType, ) -> str: return getattr(schema_module, class_name).class_class_curie @@ -161,8 +158,8 @@ _existing_layers = {} def SchemaTypeLayer( # noqa: N802 - backend: StorageBackend, - schema: str, + backend: StorageBackend, + schema: str, ) -> _SchemaTypeLayer: existing_layer, _ = _existing_layers.get(id(backend), (None, None)) if not existing_layer: diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index 5dd8523..38eab9e 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -62,8 +62,8 @@ if TYPE_CHECKING: __all__ = [ - '_SQLiteBackend', 'SQLiteBackend', + '_SQLiteBackend', ] logger = logging.getLogger('dump_things_service') @@ -88,19 +88,19 @@ class Thing(Base): class SQLResultList(BackendResultList): def __init__( - self, - engine: Any, + self, + engine: Any, ): super().__init__() self.engine = engine def generate_result( - self, - _: int, - iri: str, - class_name: str, - sort_key: str, - db_id: int, + self, + _: int, + iri: str, + class_name: str, + sort_key: str, + db_id: int, ) -> RecordInfo: """ Generate a JSON representation of the record at index `index`. @@ -124,11 +124,11 @@ class SQLResultList(BackendResultList): class _SQLiteBackend(StorageBackend): def __init__( - self, - db_path: Path, - *, - order_by: Iterable[str] | None = None, - echo: bool = False, + self, + db_path: Path, + *, + order_by: Iterable[str] | None = None, + echo: bool = False, ) -> None: assert db_path.is_absolute(), f'db_path not absolute {db_path}' if db_path.exists(): @@ -139,9 +139,7 @@ class _SQLiteBackend(StorageBackend): self.engine = create_engine('sqlite:///' + str(db_path), echo=echo) Base.metadata.create_all(self.engine) - def get_uri( - self - ) -> str: + def get_uri(self) -> str: return f'sqlite://{self.db_path}' def perform_file_name_conversion(self): @@ -152,7 +150,9 @@ class _SQLiteBackend(StorageBackend): logger.info('converting old style name %s', str(old_path)) # Create a backup copy - old_backup_path = (self.db_path.parent / (old_record_file_name + '.backup')).absolute() + old_backup_path = ( + self.db_path.parent / (old_record_file_name + '.backup') + ).absolute() logger.info('copying %s to %s', old_path, old_backup_path) shutil.copyfile(str(old_path), str(old_backup_path)) @@ -161,10 +161,10 @@ class _SQLiteBackend(StorageBackend): shutil.move(str(old_path), str(self.db_path)) def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): with Session(self.engine) as session, session.begin(): self._add_record_with_session( @@ -175,8 +175,8 @@ class _SQLiteBackend(StorageBackend): ) def add_records_bulk( - self, - record_infos: Iterable[RecordInfo], + self, + record_infos: Iterable[RecordInfo], ): with Session(self.engine) as session, session.begin(): for record_info in record_infos: @@ -188,8 +188,8 @@ class _SQLiteBackend(StorageBackend): ) def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: statement = delete(Thing).where(Thing.iri == iri) with Session(self.engine) as session, session.begin(): @@ -197,11 +197,11 @@ class _SQLiteBackend(StorageBackend): return result.rowcount == 1 def _add_record_with_session( - self, - session: Session, - iri: str, - class_name: str, - json_object: dict, + self, + session: Session, + iri: str, + class_name: str, + json_object: dict, ): sort_key = create_sort_key(json_object, self.order_by) existing_record = session.query(Thing).filter_by(iri=iri).first() @@ -220,8 +220,8 @@ class _SQLiteBackend(StorageBackend): ) def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: with Session(self.engine) as session, session.begin(): statement = select(Thing).filter_by(iri=iri) @@ -236,25 +236,24 @@ class _SQLiteBackend(StorageBackend): return None def get_records_of_classes( - self, - class_names: Iterable[str], - pattern: str | None = None, + self, + class_names: Iterable[str], + pattern: str | None = None, ) -> SQLResultList: - class_list = ', '.join(f"'{cn}'" for cn in class_names) if pattern is None: statement = text( 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'from thing ' - f"where thing.class_name in ({class_list}) " - "ORDER BY thing.sort_key" + f'where thing.class_name in ({class_list}) ' + 'ORDER BY thing.sort_key' ) else: statement = text( 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'from thing, json_tree(thing.object) ' 'where lower(json_tree.value) like lower(:pattern) ' - f"and thing.class_name in ({class_list}) " + f'and thing.class_name in ({class_list}) ' "and json_tree.type = 'text' ORDER BY thing.sort_key" ) @@ -271,14 +270,14 @@ class _SQLiteBackend(StorageBackend): ) def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> SQLResultList: if pattern is None: statement = text( 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'from thing ' - "ORDER BY thing.sort_key" + 'ORDER BY thing.sort_key' ) else: statement = text( @@ -306,7 +305,7 @@ _existing_sqlite_backends = {} def SQLiteBackend( # noqa: N802 - db_path: Path, *, order_by: Iterable[str] | None = None, echo: bool = False + db_path: Path, *, order_by: Iterable[str] | None = None, echo: bool = False ) -> _SQLiteBackend: existing_backend = _existing_sqlite_backends.get(db_path) if not existing_backend: diff --git a/dump_things_service/backends/tests/test_record_dir.py b/dump_things_service/backends/tests/test_record_dir.py index 7b44256..7d4d692 100644 --- a/dump_things_service/backends/tests/test_record_dir.py +++ b/dump_things_service/backends/tests/test_record_dir.py @@ -20,9 +20,7 @@ def test_add_and_delete_record(tmp_path): record_dir_store.build_index(str(schema_path)) record_dir_store.add_record( - iri=iri, - class_name='Object', - json_object={'pid': 'some-pid'} + iri=iri, class_name='Object', json_object={'pid': 'some-pid'} ) record = record_dir_store.get_record_by_iri(iri=iri) diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 71c1533..87a31ec 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -2,13 +2,19 @@ import logging import os import shutil from pathlib import Path -from typing import Any + +# This following lines are required for dynamic endpoint generation +from typing import ( + Annotated, # noqa: F401 -- used by autogenerated code + Any, +) from datalad_core.runners import ( - call_git_oneline, CommandError, + call_git_oneline, ) from fastapi import ( + Body, # noqa: F401 -- used by autogenerated code Depends, FastAPI, HTTPException, @@ -24,40 +30,49 @@ from starlette.responses import ( ) from dump_things_service import ( - Format, HTTP_400_BAD_REQUEST, HTTP_403_FORBIDDEN, HTTP_422_UNPROCESSABLE_CONTENT, + Format, ) from dump_things_service.abstract_config import ( CollectionConfig, - Configuration, ConfigAuthSpec, + Configuration, ForgejoAuthSpec, RecordDirBackendConfig, SQLiteBackendConfig, - read_config, check_collection, + read_config, +) +from dump_things_service.api_key import ( + api_key_header_scheme, ) from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.auth.config import ConfigAuthenticationSource from dump_things_service.auth.forgejo import ForgejoAuthenticationSource from dump_things_service.backends.record_dir_index import index_file_name from dump_things_service.backends.sqlite import record_file_name as sqlite_db_filename +from dump_things_service.converter import FormatConverter +from dump_things_service.curated import ( + store_curated_record, # noqa: F401 -- used by autogenerated code +) +from dump_things_service.exceptions import ( + ConfigCollisionError, + ConfigError, + CurieResolutionError, +) +from dump_things_service.incoming import ( + store_incoming_record, # noqa: F401 -- used by autogenerated code +) from dump_things_service.instance_state import ( InstanceState, InstanceStateCollectionInfo, - get_record_dir_config, get_instance_state, + get_record_dir_config, get_schema_info, record_dir_config_file_name, ) -from dump_things_service.converter import FormatConverter -from dump_things_service.exceptions import ( - ConfigError, - ConfigCollisionError, - CurieResolutionError, -) from dump_things_service.model import get_model_for_schema from dump_things_service.utils import ( combine_ttl, @@ -67,16 +82,9 @@ from dump_things_service.utils import ( var_escape, wrap_http_exception, ) - - -# This following lines are required for dynamic endpoint generation -from typing import Annotated # noqa 401 -- used by autogenerated code -from fastapi import Body # noqa 401 -- used by autogenerated code -from dump_things_service.api_key import api_key_header_scheme # noqa 401 -- used by autogenerated code -from dump_things_service.curated import store_curated_record # noqa 401 -- used by autogenerated code -from dump_things_service.incoming import store_incoming_record # noqa 401 -- used by autogenerated code -from dump_things_service.validate import validate_record # noqa 401 -- used by autogenerated code - +from dump_things_service.validate import ( + validate_record, # noqa: F401 -- used by autogenerated code +) logger = logging.getLogger('dump_things_service') @@ -135,9 +143,9 @@ async def {name}( def create_collection( - instance_state: InstanceState, - configuration: Configuration, - collection_name: str, + instance_state: InstanceState, + configuration: Configuration, + collection_name: str, ): """Create a collection instance as specified by `collection_configuration` @@ -191,7 +199,7 @@ def create_collection( audit_path.mkdir(parents=True) created_directories.append(audit_path) - except ConfigError as e: + except ConfigError: # Delete all directories that were created in this for directory in created_directories: shutil.rmtree(directory) @@ -222,7 +230,7 @@ def create_collection( active_classes -= set(collection_configuration.ignore_classes) instance_state.collections[collection_name] = InstanceStateCollectionInfo( active_classes=active_classes, - tag_info=dict(), + tag_info={}, ) # Create a validator for the collection @@ -262,10 +270,10 @@ def create_collection( def create_authentication_source( - abstract_configuration: Configuration, - collection_name: str, - authentication_spec: ConfigAuthSpec | ForgejoAuthSpec, - instance_state: InstanceState, + abstract_configuration: Configuration, + collection_name: str, + authentication_spec: ConfigAuthSpec | ForgejoAuthSpec, + instance_state: InstanceState, ): if collection_name not in instance_state.auth_sources: instance_state.auth_sources[collection_name] = [] @@ -293,15 +301,16 @@ def create_authentication_source( def write_record_dir_config( - path: Path, - backend_config: RecordDirBackendConfig, - schema: str, + path: Path, + backend_config: RecordDirBackendConfig, + schema: str, ): assert isinstance(backend_config, RecordDirBackendConfig) record_dir_config_file_path = path / record_dir_config_file_name if not record_dir_config_file_path.exists(): - record_dir_config_file_path.write_text(f"""# RecordDir Config + record_dir_config_file_path.write_text( + f"""# RecordDir Config type: records version: 1 schema: {schema} @@ -312,9 +321,9 @@ idfx: {backend_config.mapping_method} def check_store_compatibility( - store_path: Path, - backend_config: RecordDirBackendConfig | SQLiteBackendConfig, - schema: str, + store_path: Path, + backend_config: RecordDirBackendConfig | SQLiteBackendConfig, + schema: str, ): """Check if an existing store is compatible with the specs in `backend_config` @@ -336,24 +345,24 @@ def check_store_compatibility( def check_record_dir_compatibility( - store_path: Path, - backend_config: RecordDirBackendConfig, - schema: str, + store_path: Path, + backend_config: RecordDirBackendConfig, + schema: str, ): - # Non-existing or empty record_dir-directories are compatible if not store_path.exists(): return # A record_dir-directory is considered to be empty, if it contains no # files or only an record_dir-index file - files_in_dir = tuple(map(lambda dir_entry: dir_entry.name, os.scandir(store_path))) + files_in_dir = tuple(dir_entry.name for dir_entry in os.scandir(store_path)) if files_in_dir in ((), (index_file_name,)): return record_dir_config = get_record_dir_config(store_path) if record_dir_config.schema_location != schema: - raise ConfigCollisionError(f"Existing backend uses a different schema: '{record_dir_config.schema_location}'") + msg = f"Existing backend uses a different schema: '{record_dir_config.schema_location}'" + raise ConfigCollisionError(msg) stored_mapping_method = record_dir_config.idfx.value if stored_mapping_method != backend_config.mapping_method: @@ -363,16 +372,16 @@ def check_record_dir_compatibility( def check_sqlite_compatibility( - store_path: Path, + store_path: Path, ): sqlite_db_path = Path(store_path / sqlite_db_filename) if not sqlite_db_path.exists(): - raise ConfigError('No sqlite database found in existing store') - return + msg = 'No sqlite database found in existing store' + raise ConfigError(msg) def check_git_audit_compatibility( - audit_path: Path, + audit_path: Path, ): """Check if an existing audit path is compatible with a git audit store @@ -394,26 +403,28 @@ def check_git_audit_compatibility( force_c_locale=True, ) except CommandError as ce: - raise ConfigError(f'No git repository in gitaudit-path: {audit_path}') from ce + msg = f'No git repository in gitaudit-path: {audit_path}' + raise ConfigError(msg) from ce if result.strip().lower() != 'true': - raise ConfigError(f'No bare git repository in gitaudit-path: {audit_path}') + msg = f'No bare git repository in gitaudit-path: {audit_path}' + raise ConfigError(msg) return def create_endpoint( - operation_name: str, - operation_path: str, - instance_state: InstanceState, - collection_name: str, - collection_config: CollectionConfig, - template: str, - handler: str, - tag_group: str, - tag_name: str, - app: FastAPI, + operation_name: str, + operation_path: str, + instance_state: InstanceState, + collection_name: str, + collection_config: CollectionConfig, + template: str, + handler: str, + tag_group: str, + tag_name: str, + app: FastAPI, ): logger.info( - f'Creating %s-endpoints for collection: "%s"', + 'Creating %s-endpoints for collection: "%s"', operation_name, collection_name, ) @@ -421,12 +432,16 @@ def create_endpoint( instance_state.collections[collection_name].tag_info[tag_group] = tag_name # TODO: get schema_info from instance_state!? - model, classes, model_var_name = get_model_for_schema(collection_config.schema_location) + model, _classes, model_var_name = get_model_for_schema( + collection_config.schema_location + ) globals()[model_var_name] = model active_classes = instance_state.collections[collection_name].active_classes for class_name in active_classes: - endpoint_name = f'_endpoint_{var_escape(collection_name)}_{operation_name}_{class_name}' + endpoint_name = ( + f'_endpoint_{var_escape(collection_name)}_{operation_name}_{class_name}' + ) endpoint_source = template.format( name=endpoint_name, model_var_name=model_var_name, @@ -435,7 +450,7 @@ def create_endpoint( info=f"'{operation_name} {collection_name}/{class_name} objects'", handler=handler, ) - exec(endpoint_source, globals()) # noqa S102 + exec(endpoint_source, globals()) # noqa: S102 # Create an API route for the endpoint app.add_api_route( @@ -444,7 +459,7 @@ def create_endpoint( methods=['POST'], name=f'{operation_name} "{class_name}" object (schema: {model.linkml_meta["id"]})', response_model=None, - tags=[tag_name] + tags=[tag_name], ) logger.info( @@ -455,23 +470,51 @@ def create_endpoint( def create_endpoints_for_collection( - instance_state: InstanceState, - collection_name: str, - collection_config: CollectionConfig, - app: FastAPI, + instance_state: InstanceState, + collection_name: str, + collection_config: CollectionConfig, + app: FastAPI, ): for ( - operation_name, - operation_path, - template, - handler, - tag_group, - tag_name, + operation_name, + operation_path, + template, + handler, + tag_group, + tag_name, ) in ( - ('store', 'record', _endpoint_template, 'store_record', 'write', f'Write records to collection "{collection_name}"'), - ('validate', 'validate/record', _endpoint_template, 'validate_record', 'validate', f'Validate records for collection "{collection_name}"'), - ('curated', 'curated/record', _endpoint_curated_template, 'store_curated_record', 'curated_write', f'Curated area: store records in curated area of collection "{collection_name}"'), - ('incoming', 'incoming/{label}/record', _endpoint_incoming_template, 'store_incoming_record', 'incoming_write', f'Incoming area: store records in incoming area "{{label}}" of collection "{collection_name}"'), + ( + 'store', + 'record', + _endpoint_template, + 'store_record', + 'write', + f'Write records to collection "{collection_name}"', + ), + ( + 'validate', + 'validate/record', + _endpoint_template, + 'validate_record', + 'validate', + f'Validate records for collection "{collection_name}"', + ), + ( + 'curated', + 'curated/record', + _endpoint_curated_template, + 'store_curated_record', + 'curated_write', + f'Curated area: store records in curated area of collection "{collection_name}"', + ), + ( + 'incoming', + 'incoming/{label}/record', + _endpoint_incoming_template, + 'store_incoming_record', + 'incoming_write', + f'Incoming area: store records in incoming area "{{label}}" of collection "{collection_name}"', + ), ): create_endpoint( operation_name=operation_name, @@ -488,17 +531,16 @@ def create_endpoints_for_collection( def delete_endpoints_for_collection( - instance_state: InstanceState, - collection_name: str, + instance_state: InstanceState, + collection_name: str, ): - active_classes = instance_state.collections[collection_name].active_classes for operation_path in ( - 'record', - 'validate/record', - 'curated/record', - 'incoming/{label}/record' + 'record', + 'validate/record', + 'curated/record', + 'incoming/{label}/record', ): delete_endpoint( collection_name=collection_name, @@ -509,17 +551,17 @@ def delete_endpoints_for_collection( def delete_endpoint( - collection_name: str, - active_classes: set[str], - operation_path: str, - app: FastAPI, + collection_name: str, + active_classes: set[str], + operation_path: str, + app: FastAPI, ): from fastapi.routing import _IncludedRouter - remove_paths_set = set( + remove_paths_set = { f'/{collection_name}/{operation_path}/{class_name}' for class_name in active_classes - ) + } remove_indices = [ index @@ -532,13 +574,13 @@ def delete_endpoint( def store_record( - collection: str, - data: BaseModel | str, - class_name: str, - model: Any, - input_format: Format, - add_submission_tag: bool, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + data: BaseModel | str, + class_name: str, + model: Any, + input_format: Format, + add_submission_tag: bool, + api_key: str | None = Depends(api_key_header_scheme), ) -> JSONResponse | PlainTextResponse: if input_format == Format.json and isinstance(data, str): raise HTTPException( @@ -584,18 +626,32 @@ def store_record( ) if input_format == Format.ttl: - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Conversion error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Conversion error', + ): json_object = FormatConverter( abstract_config.collections[collection].schema_location, input_format=Format.ttl, output_format=Format.json, ).convert(data, class_name) - with wrap_http_exception(ValidationError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): - record = TypeAdapter(getattr(model, class_name)).validate_python(json_object) + with wrap_http_exception( + ValidationError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): + record = TypeAdapter(getattr(model, class_name)).validate_python( + json_object + ) else: record = data - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(record) with wrap_http_exception(CurieResolutionError): diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index 5972996..451d492 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -3,7 +3,7 @@ from pathlib import ( Path, PurePosixPath, ) -from typing import Literal +from typing import Annotated, Literal from urllib.parse import quote from fastapi import ( @@ -22,19 +22,19 @@ from dump_things_service import ( reserved_collection_names, ) from dump_things_service.abstract_config import ( - Configuration, CollectionConfig, + Configuration, + get_config, + get_token_permissions, store_config, - get_config, get_token_permissions, ) from dump_things_service.admin import authenticate_admin from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.instance_state import get_instance_state, InstanceState -from dump_things_service.manifest import manifest_configuration from dump_things_service.exceptions import ConfigError +from dump_things_service.instance_state import InstanceState, get_instance_state +from dump_things_service.manifest import manifest_configuration from dump_things_service.utils import wrap_http_exception - logger = logging.getLogger('dump_things_service') router = APIRouter() @@ -69,9 +69,9 @@ class CollectionRequest(CollectionConfig): status_code=HTTP_201_CREATED, ) async def create_collection( - response: Response, - body: CollectionRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: CollectionRequest, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): await create_or_replace_collection(body, api_key, allow_replace=False) response.headers['Location'] = f'/collections/{quote(body.name)}' @@ -84,20 +84,19 @@ async def create_collection( status_code=HTTP_201_CREATED, ) async def replace_collection( - response: Response, - body: CollectionRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: CollectionRequest, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): await create_or_replace_collection(body, api_key, allow_replace=True) response.headers['Location'] = f'/collections/{quote(body.name)}' async def create_or_replace_collection( - body: CollectionRequest, - api_key: str, - allow_replace: bool, + body: CollectionRequest, + api_key: str, + allow_replace: bool, ): - instance_state = get_instance_state() abstract_config = get_config() @@ -165,9 +164,8 @@ async def create_or_replace_collection( name='Get existing collections', ) async def get_collections( - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> list[CollectionRequest]: - instance_state = get_instance_state() abstract_config = get_config() @@ -177,7 +175,7 @@ async def get_collections( CollectionRequest( **{ 'name': collection_name, - **collection_info.model_dump(mode='json', by_alias=True) + **collection_info.model_dump(mode='json', by_alias=True), } ) for collection_name, collection_info in abstract_config.collections.items() @@ -190,10 +188,9 @@ async def get_collections( name='Get existing collection by name', ) async def get_collection_with_name( - collection_name: str, - api_key: str = Depends(api_key_header_scheme), + collection_name: str, + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> CollectionConfig: - instance_state = get_instance_state() abstract_config = get_config() @@ -214,10 +211,9 @@ async def get_collection_with_name( name='Delete collection with name', ) async def delete_collection( - collection_name: str, - api_key: str = Depends(api_key_header_scheme), + collection_name: str, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): - instance_state = get_instance_state() abstract_config = get_config() @@ -245,14 +241,16 @@ async def delete_collection( def ensure_unique_directory( - abstract_config: Configuration, - instance_state: InstanceState, - existing_dir: PurePosixPath, + abstract_config: Configuration, + instance_state: InstanceState, + existing_dir: PurePosixPath, ): abs_existing_dir = (instance_state.store_path / Path(existing_dir)).absolute() for collection_name, collection_config in abstract_config.collections.items(): for collection_dir in collection_config.curated, collection_config.incoming: - abs_collection_dir = (instance_state.store_path / Path(collection_dir)).absolute() + abs_collection_dir = ( + instance_state.store_path / Path(collection_dir) + ).absolute() if abs_collection_dir == abs_existing_dir: raise HTTPException( status_code=HTTP_409_CONFLICT, @@ -261,8 +259,8 @@ def ensure_unique_directory( def validate_incoming_paths( - abstract_config: Configuration, - collection_request: CollectionRequest, + abstract_config: Configuration, + collection_request: CollectionRequest, ): for token_name, token_info in abstract_config.tokens.items(): token_collection_info = token_info.collections.get(collection_request.name) @@ -273,7 +271,7 @@ def validate_incoming_paths( detail = ( f"Cannot add collection '{collection_request.name}' without " f"`incoming` path, because at least token '{token_name}' " - f" has write access to the collection" + f' has write access to the collection' ) raise HTTPException( status_code=HTTP_406_NOT_ACCEPTABLE, diff --git a/dump_things_service/commands/check_pids.py b/dump_things_service/commands/check_pids.py index 620e647..12a44a0 100644 --- a/dump_things_service/commands/check_pids.py +++ b/dump_things_service/commands/check_pids.py @@ -2,8 +2,8 @@ from __future__ import annotations import sys from argparse import ArgumentParser -from collections.abc import Iterable from pathlib import Path +from typing import TYPE_CHECKING from fastapi import FastAPI @@ -16,16 +16,20 @@ from dump_things_service.backends.sqlite import _SQLiteBackend from dump_things_service.exceptions import CurieResolutionError from dump_things_service.instance_state import create_instance_state from dump_things_service.manifest import manifest_configuration -from dump_things_service.store.model_store import _ModelStore from dump_things_service.utils import ( create_token_store, get_on_disk_labels, ) +if TYPE_CHECKING: + from collections.abc import Iterable + + from dump_things_service.store.model_store import _ModelStore + parser = ArgumentParser( prog='Check pids for resolvability', description='This command checks for pids that are in CURIE format and ' - 'cannot be resolved.', + 'cannot be resolved.', ) parser.add_argument( 'store', @@ -33,19 +37,7 @@ parser.add_argument( ) -def show_backend(model_store: _ModelStore): - backend = model_store.backend - if isinstance(backend, _SchemaTypeLayer): - backend = backend.backend - if isinstance(backend, _SQLiteBackend): - print(f'Checking: {backend.db_path}', file=sys.stderr) - else: - print(f'Checking: {backend.root}', file=sys.stderr) - - -def check_pids_in_stores( - stores: Iterable[_ModelStore] -) -> int: +def check_pids_in_stores(stores: Iterable[_ModelStore]) -> int: result = 0 for store in stores: print('checking', store.get_uri(), file=sys.stderr) @@ -55,13 +47,11 @@ def check_pids_in_stores( store.pid_to_iri(pid) except CurieResolutionError: result += 1 - print(pid, store.get_uri()) - return result def check_pids( - store_path: Path, + store_path: Path, ): abstract_config = read_config(store_path) instance_state = create_instance_state( @@ -94,7 +84,7 @@ def check_pids( abstract_config, instance_state, collection, - instance_state.store_path / collection_info.incoming / label + instance_state.store_path / collection_info.incoming / label, ) for label in all_labels ] diff --git a/dump_things_service/commands/copy_store.py b/dump_things_service/commands/copy_store.py index 8eeda94..27c2d61 100644 --- a/dump_things_service/commands/copy_store.py +++ b/dump_things_service/commands/copy_store.py @@ -5,6 +5,7 @@ from argparse import ArgumentParser from pathlib import Path from typing import TYPE_CHECKING +from dump_things_service.abstract_config import get_backend_and_extension from dump_things_service.backends.record_dir import ( RecordDirStore, _RecordDirStore, @@ -17,7 +18,6 @@ from dump_things_service.backends.sqlite import ( from dump_things_service.backends.sqlite import ( record_file_name as sqlite_record_file_name, ) -from dump_things_service.abstract_config import get_backend_and_extension if TYPE_CHECKING: from dump_things_service.backends import StorageBackend diff --git a/dump_things_service/commands/create_merged_schema.py b/dump_things_service/commands/create_merged_schema.py index 859a413..9ace375 100644 --- a/dump_things_service/commands/create_merged_schema.py +++ b/dump_things_service/commands/create_merged_schema.py @@ -5,21 +5,16 @@ import yaml from linkml_runtime.utils.schemaview import SchemaView # Patch linkml -from dump_things_service.patches import enabled # noqa F401 -- patches LinkML +from dump_things_service.patches import enabled # noqa: F401 -- patches LinkML parser = ArgumentParser( prog='Create a static schema with all imported schemas integrated', ) -parser.add_argument( - 'schema', - help='File containing a schema definition' -) +parser.add_argument('schema', help='File containing a schema definition') def update_uris_for_elements( - all_elements: dict, - attribute_name: str, - prefix_index: dict + all_elements: dict, attribute_name: str, prefix_index: dict ): for name, info in all_elements.items(): uri = getattr(info, attribute_name) @@ -32,7 +27,7 @@ def update_uris_for_elements( def update_uris(schema_view: SchemaView): - """ Update element-defining URIs to the original element source + """Update element-defining URIs to the original element source Element-defining URIs (e.g., slot_uri, class_uri) are by default set to the schema in which the element is defined. In this case, that would be the @@ -68,7 +63,7 @@ def main(): Dumper=yaml.SafeDumper, allow_unicode=True, sort_keys=False, - ) + ) print(text) return 0 diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py index f63b098..698d6b2 100644 --- a/dump_things_service/commands/download_config.py +++ b/dump_things_service/commands/download_config.py @@ -8,37 +8,38 @@ from argparse import ArgumentParser import requests import yaml - parser = ArgumentParser( prog='Download a complete configuration of a running service', description='Read a configuration from dump-things endpoints and create a ' - 'configuration-file that can be possibly modified and uploaded ' - 'to a running service by dump-things-upload-config.' - ' ' - 'An admin token has to be provided in the environment variable ' - '`DTS_ADMIN_TOKEN`.', + 'configuration-file that can be possibly modified and uploaded ' + 'to a running service by dump-things-upload-config.' + ' ' + 'An admin token has to be provided in the environment variable ' + '`DTS_ADMIN_TOKEN`.', ) parser.add_argument( 'server_api', help='The base URL of the server API.', ) parser.add_argument( - '--entities', '-e', + '--entities', + '-e', action='append', choices=['admin_tokens', 'collections', 'tokens'], help='Specify for which entities the configuration should be downloaded. ' - ' Possible values are `admin_tokens`, `collections`, or `tokens` ' - '(repeat to download configuration for more than one entity). If this ' - 'option is not provided, configurations for all entities will be ' - 'downloaded.' + ' Possible values are `admin_tokens`, `collections`, or `tokens` ' + '(repeat to download configuration for more than one entity). If this ' + 'option is not provided, configurations for all entities will be ' + 'downloaded.', ) parser.add_argument( - '--format', '-f', + '--format', + '-f', nargs='?', default='yaml', choices=['json', 'yaml'], help='Specify the format of the output. Possible values are `json` ' - 'and `yaml` (the default is `yaml`).' + 'and `yaml` (the default is `yaml`).', ) @@ -86,11 +87,10 @@ def main(): def get_configuration( - api_url: str, - admin_token: str, - entities: list[str], + api_url: str, + admin_token: str, + entities: list[str], ) -> dict: - result = {} if 'collections' in entities: @@ -110,12 +110,13 @@ def get_configuration( def list_to_dict_on_key( - elements: list[dict], - extract_key: str, + elements: list[dict], + extract_key: str, ) -> dict: return { element[extract_key]: { - element_key: value for element_key, value in element.items() + element_key: value + for element_key, value in element.items() if element_key != extract_key } for element in elements @@ -123,8 +124,8 @@ def list_to_dict_on_key( def get_tokens( - api_url: str, - admin_token: str, + api_url: str, + admin_token: str, ) -> dict: token_list = _get_data( url=api_url + '/tokens', @@ -135,8 +136,8 @@ def get_tokens( def get_collections( - api_url: str, - admin_token: str, + api_url: str, + admin_token: str, ) -> dict: collection_list = _get_data( url=api_url + '/collections', @@ -147,8 +148,8 @@ def get_collections( def get_admin_tokens( - api_url: str, - admin_token: str, + api_url: str, + admin_token: str, ) -> dict: admin_token_list = _get_data( url=api_url + '/admin_tokens', @@ -164,9 +165,9 @@ def get_admin_tokens( def _get_data( - url: str, - token: str, - content_class: str, + url: str, + token: str, + content_class: str, ) -> list: result = requests.get(url, headers={'x-dumpthings-token': token}) if result.status_code >= 300: diff --git a/dump_things_service/commands/gitaudit_rebuild_index.py b/dump_things_service/commands/gitaudit_rebuild_index.py index bde297e..30e8489 100644 --- a/dump_things_service/commands/gitaudit_rebuild_index.py +++ b/dump_things_service/commands/gitaudit_rebuild_index.py @@ -6,14 +6,12 @@ from pathlib import Path from dump_things_service.audit.gitaudit import GitAuditBackend - parser = ArgumentParser( prog='Rebuild the index of a `gitaudit`-database', - description='This command rebuilds the index of a `gitaudit`-database.' + description='This command rebuilds the index of a `gitaudit`-database.', ) parser.add_argument( - 'audit_store', - help='The directory in which the `gitaudit`-database is located.' + 'audit_store', help='The directory in which the `gitaudit`-database is located.' ) diff --git a/dump_things_service/commands/gitaudit_report.py b/dump_things_service/commands/gitaudit_report.py index 06600b8..a117e36 100644 --- a/dump_things_service/commands/gitaudit_report.py +++ b/dump_things_service/commands/gitaudit_report.py @@ -8,12 +8,11 @@ from pathlib import Path from dump_things_service.audit.gitaudit import GitAuditBackend - parser = ArgumentParser( prog='Report audit information for a PID', description='Report the audit information that was stored for a specific ' - 'PID. For every change to a record the tool will report: ' - 'time stamp, user ID, diff, and the resulting record.', + 'PID. For every change to a record the tool will report: ' + 'time stamp, user ID, diff, and the resulting record.', ) parser.add_argument( 'audit_store', @@ -22,8 +21,8 @@ parser.add_argument( parser.add_argument( 'pid', help='Regex pattern that identifies PIDs of the record for which audit ' - 'information should be reported ' - '(to see all audit log entries, specify ".*").', + 'information should be reported ' + '(to see all audit log entries, specify ".*").', ) diff --git a/dump_things_service/commands/hash_token.py b/dump_things_service/commands/hash_token.py index 5f5d478..9ced0cb 100644 --- a/dump_things_service/commands/hash_token.py +++ b/dump_things_service/commands/hash_token.py @@ -5,12 +5,11 @@ from argparse import ArgumentParser from dump_things_service.abstract_config import hash_token_representation - parser = ArgumentParser( prog='Hash a plain text token to create a hashed token in a dump-things server', description='Hash a token and print the calculated hash value. The hash value ' - 'can be used to create a hashed token via the `/tokens`-endpoint ' - 'of a dump-things-server.', + 'can be used to create a hashed token via the `/tokens`-endpoint ' + 'of a dump-things-server.', ) parser.add_argument( 'token', @@ -23,12 +22,13 @@ def main(): arguments = parser.parse_args() token = arguments.token.strip() - if any(map(lambda s: s.isspace(), token)): + if any(s.isspace() for s in token): print('Whitespace are not allowed in token', file=sys.stderr, flush=True) return 1 print(hash_token_representation(token)) return 0 + if __name__ == '__main__': sys.exit(main()) diff --git a/dump_things_service/commands/rebuild_index.py b/dump_things_service/commands/rebuild_index.py index 0825dc4..09846e2 100644 --- a/dump_things_service/commands/rebuild_index.py +++ b/dump_things_service/commands/rebuild_index.py @@ -7,9 +7,8 @@ from pathlib import Path import yaml from dump_things_service import config_file_name -from dump_things_service.backends.record_dir_index import RecordDirIndex from dump_things_service.abstract_config import RecordDirConfigFileContent - +from dump_things_service.backends.record_dir_index import RecordDirIndex parser = ArgumentParser( prog='Rebuild the index of a `record_dir`-store', diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index 71a2395..6955e96 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -12,52 +12,52 @@ import yaml from dump_things_service.instance_state import get_record_dir_config - parser = ArgumentParser( prog='Establish a configuration in a running service', description='Read a configuration from a dump-things configuration-file ' - 'and instantiate its elements on a running server. Objects that ' - 'already exist on the server are left unchanged. ' - ' ' - 'An admin token has to be provided in the environment variable ' - '`DTS_ADMIN_TOKEN`.', + 'and instantiate its elements on a running server. Objects that ' + 'already exist on the server are left unchanged. ' + ' ' + 'An admin token has to be provided in the environment variable ' + '`DTS_ADMIN_TOKEN`.', ) parser.add_argument( 'config_file', help='The path to the config file', ) parser.add_argument( - '--format', '-f', + '--format', + '-f', nargs='?', choices=['json', 'yaml'], help='Specify the format of the input file. Possible values are `json` ' - 'and `yaml`. If this option is given, the ' - 'suffix of the configuration file is ignored.' + 'and `yaml`. If this option is given, the ' + 'suffix of the configuration file is ignored.', ) parser.add_argument( '--send-to', help='The base URL of the server API. If this option is provided, the ' - 'configuration will be sent to the server API, otherwise it will just ' - 'be written to stdout.', + 'configuration will be sent to the server API, otherwise it will just ' + 'be written to stdout.', ) parser.add_argument( '--old-format', action='store_true', help='If provided, assume that the configuration is in version 1 format ' - 'and convert it to the new format internally (in version 1: tokens ' - 'had no `hashed`-attribute and no `representation`-attribute, the token ' - 'representation was the key of the token configuration, ' - 'collections had no `schema`-attribute, and `sqlite`-backends had ' - 'a `schema`-attribute).', + 'and convert it to the new format internally (in version 1: tokens ' + 'had no `hashed`-attribute and no `representation`-attribute, the token ' + 'representation was the key of the token configuration, ' + 'collections had no `schema`-attribute, and `sqlite`-backends had ' + 'a `schema`-attribute).', ) parser.add_argument( '--store', default=None, help='If --old-format is provided, this option can be used to specify a ' - 'store directory. The store directory will be used to load `RecordDir` ' - 'configurations, if a collection defines are `RecordDir`-backend. ' - '(This option has no effect if no collection in the old configuration ' - 'uses a `RecordDir`-backend.)', + 'store directory. The store directory will be used to load `RecordDir` ' + 'configurations, if a collection defines are `RecordDir`-backend. ' + '(This option has no effect if no collection in the old configuration ' + 'uses a `RecordDir`-backend.)', ) @@ -85,16 +85,17 @@ def main(): if arguments.old_format: configuration = convert_config_1_to_config_2(configuration, arguments.store) - else: - if arguments.store: - print( - 'Warning: ignoring `--store` option because `--old-format` ' - 'is not provided.', - file=sys.stderr, - flush=True, - ) + elif arguments.store: + print( + 'Warning: ignoring `--store` option because `--old-format` ' + 'is not provided.', + file=sys.stderr, + flush=True, + ) - assert configuration['type'] == 'collections', '`type: collections` missing in config-file' + assert configuration['type'] == 'collections', ( + '`type: collections` missing in config-file' + ) assert configuration['version'] == 2, '`version: 2` missing in config-file' if arguments.send_to: @@ -110,9 +111,7 @@ def main(): try: establish_configuration( configuration, - arguments.send_to[:-1] - if arguments.send_to.endswith('/') - else arguments.send_to, + arguments.send_to.removesuffix('/'), admin_token, ) return 0 @@ -135,10 +134,9 @@ def main(): def convert_config_1_to_config_2( - old_configuration: dict, - store_path: str | Path, + old_configuration: dict, + store_path: str | Path, ) -> dict: - old_version = old_configuration.get('version') if old_version != 1: msg = f'`Unknown old configuration format: {old_version}' @@ -154,9 +152,11 @@ def convert_config_1_to_config_2( f'token_{next(counter)}': { **old_token_config.copy(), 'representation': token_representation, - 'hashed': False + 'hashed': False, } - for token_representation, old_token_config in old_configuration['tokens'].items() + for token_representation, old_token_config in old_configuration[ + 'tokens' + ].items() } old_to_new_token_mapping = { @@ -165,7 +165,7 @@ def convert_config_1_to_config_2( } store_path = Path(store_path) if store_path else None - for collection_name, collection_config in old_configuration['collections'].items(): + for collection_config in old_configuration['collections'].values(): backend = collection_config.get('backend') if backend and backend['type'].startswith('sqlite'): collection_config['schema'] = backend['schema'] @@ -174,29 +174,32 @@ def convert_config_1_to_config_2( if store_path is None: msg = '--store has to be provided to convert collection with record_dir-backends' raise ValueError(msg) - record_dir_config = get_record_dir_config(store_path / collection_config['curated']) + record_dir_config = get_record_dir_config( + store_path / collection_config['curated'] + ) collection_config['schema'] = record_dir_config.schema_location backend = { 'type': 'record_dir+stl' if not backend else backend['type'], - 'mapping_method': record_dir_config.idfx.value + 'mapping_method': record_dir_config.idfx.value, } collection_config['backend'] = backend - collection_config['default_token'] = old_to_new_token_mapping[collection_config['default_token']] + collection_config['default_token'] = old_to_new_token_mapping[ + collection_config['default_token'] + ] - new_configuration = { + return { 'type': 'collections', 'version': 2, 'tokens': new_tokens_dict, 'collections': old_configuration['collections'], 'admin_tokens': {}, } - return new_configuration def establish_configuration( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): create_collections(configuration, api_url, admin_token) create_tokens(configuration, api_url, admin_token) @@ -204,9 +207,9 @@ def establish_configuration( def create_tokens( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): for token_name, token_config in configuration['tokens'].items(): _post_data( @@ -222,9 +225,9 @@ def create_tokens( def create_collections( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): for collection_name, collection_config in configuration['collections'].items(): _post_data( @@ -240,9 +243,9 @@ def create_collections( def create_admin_tokens( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): for admin_token_name, admin_token_config in configuration['admin_tokens'].items(): _post_data( @@ -258,13 +261,17 @@ def create_admin_tokens( def _post_data( - url: str, - data: dict, - token: str, - content_class: str, - content_name: str, + url: str, + data: dict, + token: str, + content_class: str, + content_name: str, ): - result = requests.put(url, headers={'x-dumpthings-token': token}, json=data,) + result = requests.put( + url, + headers={'x-dumpthings-token': token}, + json=data, + ) if result.status_code >= 300: msg = f'Error uploading {content_class}: {content_name}: {result.text}' raise RuntimeError(msg) diff --git a/dump_things_service/converter.py b/dump_things_service/converter.py index ed85a8d..c39686a 100644 --- a/dump_things_service/converter.py +++ b/dump_things_service/converter.py @@ -6,10 +6,8 @@ from json import loads as json_loads from typing import ( TYPE_CHECKING, Any, - Callable, ) -from linkml_runtime import SchemaView from linkml.utils.datautils import ( get_dumper, get_loader, @@ -29,10 +27,11 @@ from dump_things_service.model import ( ) from dump_things_service.utils import cleaned_json - if TYPE_CHECKING: + from collections.abc import Callable from types import ModuleType + from linkml_runtime import SchemaView from pydantic import BaseModel from dump_things_service.backends import RecordInfo @@ -47,10 +46,7 @@ class TypeValidator: self.type_name = type_name self.matcher = None if pattern is None else re.compile(pattern) - def validate( - self, - value: str - ) -> str: + def validate(self, value: str) -> str: if self.matcher: match = self.matcher.match(value) if not match: @@ -238,10 +234,7 @@ def _convert_format( ) except Exception as e: # BLE001 if load_only: - msg = ( - f'Validation error for instance of {target_class}: {e}, ' - f'data:\n{data}' - ) + msg = f'Validation error for instance of {target_class}: {e}, data:\n{data}' else: msg = ( f'Conversion {input_format} -> {output_format}. Error: {e}, ' diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index fef4529..f8a8ede 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -1,13 +1,11 @@ from __future__ import annotations import logging -from itertools import count -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated from fastapi import ( APIRouter, Depends, - FastAPI, HTTPException, ) from fastapi_pagination import ( @@ -19,14 +17,13 @@ from fastapi_pagination import ( from dump_things_service import ( HTTP_401_UNAUTHORIZED, HTTP_404_NOT_FOUND, - HTTP_422_UNPROCESSABLE_CONTENT, abstract_config, + HTTP_422_UNPROCESSABLE_CONTENT, ) from dump_things_service.abstract_config import ( check_collection, read_config, ) from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError from dump_things_service.instance_state import get_instance_state @@ -41,6 +38,7 @@ from dump_things_service.utils import ( if TYPE_CHECKING: from pydantic import BaseModel + from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends import StorageBackend from dump_things_service.lazy_list import LazyList from dump_things_service.store.model_store import _ModelStore @@ -76,7 +74,7 @@ add_pagination(router) @router.get( '/{collection}/curated/records/{class_name}', tags=['Curated area: read records'], - name='Read all records of the given class from the curated area' + name='Read all records of the given class from the curated area', ) async def read_curated_records_of_type( collection: str, @@ -104,7 +102,7 @@ async def read_curated_records_of_type( @router.get( '/{collection}/curated/records/p/{class_name}', tags=['Curated area: read records'], - name='Read all records of the given class from the curated area with pagination' + name='Read all records of the given class from the curated area with pagination', ) async def read_curated_records_of_type_paginated( collection: str, @@ -112,7 +110,6 @@ async def read_curated_records_of_type_paginated( matching: str | None = None, api_key: str | None = Depends(api_key_header_scheme), ) -> Page[dict]: - instance_state = get_instance_state() if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( @@ -133,7 +130,7 @@ async def read_curated_records_of_type_paginated( @router.get( '/{collection}/curated/records/', tags=['Curated area: read records'], - name='Read all records from the curated area' + name='Read all records from the curated area', ) async def read_curated_all_records( collection: str, @@ -153,7 +150,7 @@ async def read_curated_all_records( @router.get( '/{collection}/curated/records/p/', tags=['Curated area: read records'], - name='Read all records from the curated area with pagination' + name='Read all records from the curated area with pagination', ) async def read_curated_all_records_paginated( collection: str, @@ -174,12 +171,12 @@ async def read_curated_all_records_paginated( @router.get( '/{collection}/curated/record', tags=['Curated area: read records'], - name='Read the record with the given pid from the curated area' + name='Read the record with the given pid from the curated area', ) async def read_curated_record_with_pid( collection: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _read_curated_records( collection=collection, @@ -192,12 +189,12 @@ async def read_curated_record_with_pid( @router.delete( '/{collection}/curated/record', tags=['Curated area: delete records'], - name='Delete the record with the given pid from the curated area of the given collection' + name='Delete the record with the given pid from the curated area of the given collection', ) async def delete_curated_record_with_pid( collection: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _delete_curated_record( collection=collection, @@ -214,7 +211,6 @@ async def _read_curated_records( api_key: str | None = None, upper_bound: int | None = 1000, ) -> LazyList | dict | None: - model_store, backend, _ = _get_store_and_backend(collection, api_key) if pid: @@ -232,9 +228,7 @@ async def _read_curated_records( len(result_list), upper_bound, collection, - f'/curated/records/p/{class_name}' - if class_name - else '/curated/records/p/', + f'/curated/records/p/{class_name}' if class_name else '/curated/records/p/', ) return ModifierList( @@ -244,9 +238,9 @@ async def _read_curated_records( async def _delete_curated_record( - collection: str, - pid: str | None, - api_key: str | None = None, + collection: str, + pid: str | None, + api_key: str | None = None, ) -> bool: with wrap_http_exception(Exception): model_store, backend, _ = _get_store_and_backend(collection, api_key) @@ -255,7 +249,7 @@ async def _delete_curated_record( raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"Could not remove record with PID '{pid}' from curated area " - f"of collection '{collection}'.", + f"of collection '{collection}'.", ) return True @@ -264,7 +258,6 @@ def _get_store_and_backend( collection: str, plain_token: str | None, ) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]: - # A token is required if plain_token is None: raise HTTPException( @@ -296,14 +289,18 @@ def _get_store_and_backend( def store_curated_record( - collection: str, - data: BaseModel, - class_name: str, - author_id: str | None = None, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + data: BaseModel, + class_name: str, + author_id: str | None = None, + api_key: str | None = Depends(api_key_header_scheme), ): instance_state = get_instance_state() - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(data) pid = data.pid diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index 2d888cd..621c652 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated from fastapi import ( APIRouter, @@ -22,8 +22,8 @@ from dump_things_service import ( from dump_things_service.abstract_config import ( check_collection, check_label, - get_config_labels, get_config, + get_config_labels, ) from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer @@ -55,25 +55,27 @@ add_pagination(router) @router.get( '/{collection}/incoming/', tags=['Incoming area: read labels'], - name='Get all incoming labels for the given collection' + name='Get all incoming labels for the given collection', ) async def incoming_read_labels( collection: str, - api_key: str | None = Depends(api_key_header_scheme), + api_key: Annotated[str | None, Depends(api_key_header_scheme)], ) -> list[str]: # Authorize api_key await authorize_zones(collection, api_key) instance_state = get_instance_state() configured_labels = get_config_labels(get_config(), collection) - on_disk_labels = get_on_disk_labels(instance_state.store_path, get_config(), collection) + on_disk_labels = get_on_disk_labels( + instance_state.store_path, get_config(), collection + ) return list(configured_labels.union(on_disk_labels)) @router.get( '/{collection}/incoming/{label}/records/{class_name}', tags=['Incoming area: read records'], - name='Read all records of the given class from the given incoming area' + name='Read all records of the given class from the given incoming area', ) async def incoming_read_records_of_type( collection: str, @@ -103,7 +105,7 @@ async def incoming_read_records_of_type( @router.get( '/{collection}/incoming/{label}/records/p/{class_name}', tags=['Incoming area: read records'], - name='Read all records of the given class from the given incoming area with pagination' + name='Read all records of the given class from the given incoming area with pagination', ) async def incoming_read_records_of_type_paginated( collection: str, @@ -112,7 +114,6 @@ async def incoming_read_records_of_type_paginated( matching: str | None = None, api_key: str | None = Depends(api_key_header_scheme), ) -> Page[dict]: - instance_state = get_instance_state() if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( @@ -134,7 +135,7 @@ async def incoming_read_records_of_type_paginated( @router.get( '/{collection}/incoming/{label}/records/', tags=['Incoming area: read records'], - name='Read all records from the given incoming area' + name='Read all records from the given incoming area', ) async def incoming_read_all_records( collection: str, @@ -156,13 +157,13 @@ async def incoming_read_all_records( @router.get( '/{collection}/incoming/{label}/records/p/', tags=['Incoming area: read records'], - name='Read all records from the given incoming area with pagination' + name='Read all records from the given incoming area with pagination', ) async def incoming_read_all_records_paginated( - collection: str, - label: str, - matching: str | None = None, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + label: str, + matching: str | None = None, + api_key: str | None = Depends(api_key_header_scheme), ) -> Page[dict]: record_list = await _incoming_read_records( collection=collection, @@ -179,13 +180,13 @@ async def incoming_read_all_records_paginated( @router.get( '/{collection}/incoming/{label}/record', tags=['Incoming area: read records'], - name='Read the record with the given PID from the given incoming area' + name='Read the record with the given PID from the given incoming area', ) async def incoming_read_record_with_pid( - collection: str, - label: str, - pid: str, - api_key: str = Depends(api_key_header_scheme), + collection: str, + label: str, + pid: str, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _incoming_read_records( collection=collection, @@ -199,13 +200,13 @@ async def incoming_read_record_with_pid( @router.delete( '/{collection}/incoming/{label}/record', tags=['Incoming area: delete records'], - name='Delete the record with the given PID from the given incoming area' + name='Delete the record with the given PID from the given incoming area', ) async def incoming_delete_record_with_pid( collection: str, label: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _incoming_delete_record( collection=collection, @@ -216,15 +217,14 @@ async def incoming_delete_record_with_pid( async def _incoming_read_records( - collection: str, - label: str, - class_name: str | None, - pid: str | None, - matching: str | None = None, - api_key: str | None = None, - upper_bound: int = 1000, + collection: str, + label: str, + class_name: str | None, + pid: str | None, + matching: str | None = None, + api_key: str | None = None, + upper_bound: int = 1000, ) -> LazyList | dict | None: - model_store, backend = await _get_store_and_backend(collection, label, api_key) if pid: @@ -244,7 +244,7 @@ async def _incoming_read_records( collection, f'/incoming/{label}/records/p/{class_name}' if class_name - else f'/incoming/{label}/records/p/' + else f'/incoming/{label}/records/p/', ) return ModifierList( @@ -266,7 +266,7 @@ async def _incoming_delete_record( raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"Could not remove record with PID '{pid}' from incoming " - f"area '{label}' of collection '{collection}'.", + f"area '{label}' of collection '{collection}'.", ) return True @@ -276,7 +276,6 @@ async def _get_store_and_backend( label: str, plain_token: str | None, ) -> tuple[_ModelStore, StorageBackend]: - # Authorize api_key await authorize_zones(collection, plain_token) @@ -301,33 +300,6 @@ async def _get_store_and_backend( store_dir=store_dir, ) - xxx = """ - # For consistency, associate the store with all matching tokens from the - # configuration file. That means with all tokens that have the same - # input - matching_tokens = [ - token_name - for token_name, token_info in abstract_config.tokens.items() - if (collection, label) in [ - (collection_name, token_collection_info.incoming_label) - for collection_name, token_collection_info in token_info.items() - ] - ] - - for matching_token in matching_tokens: - # Associate the store with all matching tokens in the configuration. - # Note: there are stores that are not associated with a token in - # the abstract configuration. These are stores that belong to a token - # that is authenticated with an external authentication source. - token_info = instance_state.tokens[collection][matching_token] - instance_state.token_stores[collection][matching_token] = ( - model_store, - matching_token, - token_info['permissions'], - token_info['user_id'], - ) - """ - backend = model_store.backend if isinstance(backend, _SchemaTypeLayer): return model_store, backend.backend @@ -361,15 +333,18 @@ async def authorize_zones( async def store_incoming_record( - collection: str, - label: str, - data: BaseModel, - class_name: str, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + label: str, + data: BaseModel, + class_name: str, + api_key: str | None = Depends(api_key_header_scheme), ): - instance_state = get_instance_state() - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(data) pid = data.pid diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index 359a185..703febb 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -3,25 +3,20 @@ from __future__ import annotations import dataclasses import logging from functools import cache -from pathlib import Path -from types import ModuleType from typing import ( + TYPE_CHECKING, Any, - Callable, ) import yaml -from fastapi import FastAPI -from linkml_runtime import SchemaView from pydantic import ValidationError from yaml.scanner import ScannerError from dump_things_service.abstract_config import ( - RecordDirConfigFileContent, MappingMethod, + RecordDirConfigFileContent, mapping_functions, ) - from dump_things_service.converter import get_conversion_objects from dump_things_service.exceptions import ConfigError from dump_things_service.model import ( @@ -30,6 +25,13 @@ from dump_things_service.model import ( get_schema_view, ) +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from types import ModuleType + + from fastapi import FastAPI + from linkml_runtime import SchemaView logger = logging.getLogger('dump_things_service') @@ -86,7 +88,9 @@ class InstanceState: maintenance_mode: set = dataclasses.field(default_factory=set) # Created based on abstract configuration - collections: dict[str, InstanceStateCollectionInfo] = dataclasses.field(default_factory=dict) + collections: dict[str, InstanceStateCollectionInfo] = dataclasses.field( + default_factory=dict + ) tokens: dict = dataclasses.field(default_factory=dict) auth_sources: dict[str, list] = dataclasses.field(default_factory=dict) audit_backends: dict[str, list] = dataclasses.field(default_factory=dict) @@ -97,13 +101,13 @@ class InstanceState: order_by: list[str] = dataclasses.field(default_factory=list) -g_instance_state:InstanceState | None = None +g_instance_state: InstanceState | None = None def create_instance_state( - store_path: Path, - bootstrap_token: str, - fastapi_app: FastAPI, + store_path: Path, + bootstrap_token: str, + fastapi_app: FastAPI, ) -> InstanceState: global g_instance_state @@ -128,8 +132,8 @@ def get_instance_state() -> InstanceState: def get_record_dir_config( - path: Path, - file_name: str = record_dir_config_file_name, + path: Path, + file_name: str = record_dir_config_file_name, ) -> RecordDirConfigFileContent: config_path = path / file_name if not config_path.exists(): diff --git a/dump_things_service/lazy_list.py b/dump_things_service/lazy_list.py index 83eb0de..8a86c43 100644 --- a/dump_things_service/lazy_list.py +++ b/dump_things_service/lazy_list.py @@ -27,10 +27,9 @@ from abc import ( from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable from typing import ( Any, - Callable, ) @@ -177,7 +176,7 @@ class PriorityList(LazyList): """ def __init__( - self, + self, ): super().__init__() self.seen = set() diff --git a/dump_things_service/main.py b/dump_things_service/main.py index 7803ffc..b357563 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -5,13 +5,14 @@ import logging import os import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated from dump_things_service.abstract_config import store_config from dump_things_service.commands.upload_config import convert_config_1_to_config_2 from dump_things_service.manifest import manifest_configuration + # Perform the patching before importing any third-party libraries -from dump_things_service.patches import enabled # noqa F401 -- used by generated code +from dump_things_service.patches import enabled # noqa: F401 -- used by generated code import yaml import uvicorn @@ -57,8 +58,7 @@ from dump_things_service.converter import ( from dump_things_service.curated import router as curated_router from dump_things_service.exceptions import CurieResolutionError from dump_things_service.incoming import router as incoming_router -from dump_things_service.instance_state import create_instance_state, \ - InstanceState +from dump_things_service.instance_state import create_instance_state, InstanceState from dump_things_service.lazy_list import ( PriorityList, ModifierList, @@ -97,7 +97,7 @@ class ServerCollectionCountedResponse(ServerCollectionResponse): class ServerResponse(BaseModel): version: str - collections: list[ServerCollectionResponse|ServerCollectionCountedResponse] + collections: list[ServerCollectionResponse | ServerCollectionCountedResponse] logging.basicConfig(level=logging.WARNING) @@ -106,7 +106,7 @@ logger = logging.getLogger('dump_things_service') parser = argparse.ArgumentParser() -parser.add_argument('--host', default='0.0.0.0') # noqa S104 +parser.add_argument('--host', default='0.0.0.0') # noqa: S104 parser.add_argument('--port', default=8000, type=int) parser.add_argument('--origins', action='append', default=[]) parser.add_argument( @@ -114,19 +114,19 @@ parser.add_argument( type=str, default='', help='The sha256 hash of an initial admin token that will allow to add or ' - 'remove tokens, collections, and additional admin tokens (64 ' - 'characters hex-digit). NOTE: an admin token in plaintext is read ' - 'from the environment variable `DTS_ADMIN_TOKEN` if it is set, and ' - 'if this option is not provided.', + 'remove tokens, collections, and additional admin tokens (64 ' + 'characters hex-digit). NOTE: an admin token in plaintext is read ' + 'from the environment variable `DTS_ADMIN_TOKEN` if it is set, and ' + 'if this option is not provided.', ) parser.add_argument( '-c', '--config', metavar='CONFIG_FILE', help="Read the configuration from 'CONFIG_FILE' if no persisted " - "configuration is found in the data store root directory, and " - "initialize the persistent configuration and the service state with " - "the values in 'CONFIG_FILE'.", + 'configuration is found in the data store root directory, and ' + 'initialize the persistent configuration and the service state with ' + "the values in 'CONFIG_FILE'.", ) parser.add_argument( '--root-path', @@ -141,10 +141,10 @@ parser.add_argument( parser.add_argument( '--ignore-default-config-file', action='store_true', - help="If the persisted configuration is empty, do not try to initialize it " - "from an existing default-config file, i.e., do not read the file " - "`/.dumpthings.yaml`. That means the configuration be empty " - "collections and tokens are added via the API.", + help='If the persisted configuration is empty, do not try to initialize it ' + 'from an existing default-config file, i.e., do not read the file ' + '`/.dumpthings.yaml`. That means the configuration be empty ' + 'collections and tokens are added via the API.', ) parser.add_argument( 'store', @@ -182,15 +182,14 @@ if not arguments.admin_token_hash: arguments.admin_token_hash = hash_token_representation( os.environ.get('DTS_ADMIN_TOKEN', ''), ) -else: - # Validate the hash token format - if not hash_matcher.match(arguments.admin_token_hash): - print( - 'Hashed admin token is not a 64-digits hex-number', - file=sys.stderr, - flush=True, - ) - sys.exit(1) +# Validate the hash token format +elif not hash_matcher.match(arguments.admin_token_hash): + print( + 'Hashed admin token is not a 64-digits hex-number', + file=sys.stderr, + flush=True, + ) + sys.exit(1) # Set the log level @@ -247,8 +246,8 @@ g_configuration = read_config(store_path) def initialize_from_config_file( - instance_state: InstanceState, - config_file: str | Path, + instance_state: InstanceState, + config_file: str | Path, ) -> Configuration: with open(config_file) as f: config_dict = yaml.safe_load(f) @@ -274,20 +273,20 @@ def initialize_from_config_file( # location, i.e., from `/.dumpthings.yaml`, or from the configuration # option, unless `--dont-use-old-config` is specified. if not ( - g_configuration.admin_tokens - or g_configuration.collections - or g_configuration.tokens + g_configuration.admin_tokens + or g_configuration.collections + or g_configuration.tokens ): if arguments.config: config_file = arguments.config + elif arguments.ignore_default_config_file: + config_file = None else: - if arguments.ignore_default_config_file: + from dump_things_service import config_file_name + + config_file = g_instance_state.store_path / config_file_name + if not config_file.exists(): config_file = None - else: - from dump_things_service import config_file_name - config_file = g_instance_state.store_path / config_file_name - if not config_file.exists(): - config_file = None if config_file: logger.info( @@ -307,18 +306,17 @@ if not ( # If there are no structures in the configuration, check for a bootstrap token. if not ( - g_configuration.admin_tokens - or g_configuration.collections - or g_configuration.tokens -): - if not g_instance_state.bootstrap_token: - print( - 'The server has an empty configuration and requires a bootstrap ' - 'token (use `--admin-token-hash` to provide one).', - file=sys.stderr, - flush=True, - ) - sys.exit(2) + g_configuration.admin_tokens + or g_configuration.collections + or g_configuration.tokens +) and not g_instance_state.bootstrap_token: + print( + 'The server has an empty configuration and requires a bootstrap ' + 'token (use `--admin-token-hash` to provide one)', + file=sys.stderr, + flush=True, + ) + sys.exit(2) manifest_configuration( @@ -337,38 +335,36 @@ async def root() -> RedirectResponse: return RedirectResponse('/docs') -@app.get( - '/server', - tags=['Server management'], - name='get server information' -) +@app.get('/server', tags=['Server management'], name='get server information') async def server() -> ServerResponse: return ServerResponse( - version = __version__, - collections = [ + version=__version__, + collections=[ ServerCollectionResponse( name=collection_name, schema=g_configuration.collections[collection_name].schema_location, - classes=g_instance_state.schema_info[g_configuration.collections[collection_name].schema_location].classes, + classes=g_instance_state.schema_info[ + g_configuration.collections[collection_name].schema_location + ].classes, ) for collection_name in g_configuration.collections - ] + ], ) @app.post( '/maintenance', tags=['Server management'], - name='put a collection in maintenance mode' + name='put a collection in maintenance mode', ) async def maintenance( body: MaintenanceRequest, - api_key: str | None = Depends(api_key_header_scheme), + api_key: Annotated[str | None, Depends(api_key_header_scheme)], ): if api_key is None: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f'Token required for this operation', + detail='Token required for this operation', ) collection = body.collection @@ -381,20 +377,19 @@ async def maintenance( permissions = auth_info.token_permission if not ( - permissions.curated_write - and permissions.curated_read - and permissions.zones_access + permissions.curated_write + and permissions.curated_read + and permissions.zones_access ): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f'Curator permissions required for this operation', + detail='Curator permissions required for this operation', ) if active: g_instance_state.maintenance_mode.add(collection) else: g_instance_state.maintenance_mode.remove(collection) - return @app.get( @@ -405,7 +400,7 @@ async def maintenance( async def read_record_with_pid( collection: str, pid: str, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ): check_collection(g_configuration, collection) @@ -446,10 +441,10 @@ async def read_record_with_pid( name='Read all records from the given collection', ) async def read_all_records( - collection: str, - matching: str | None = None, - format: Format = Format.json, # noqa A002 - api_key: str = Depends(api_key_header_scheme), + collection: str, + matching: str | None = None, + format: Format = Format.json, # noqa: A002 + api_key: str = Depends(api_key_header_scheme), ): return await _read_all_records( collection=collection, @@ -469,10 +464,10 @@ async def read_all_records( name='Read all records from the given collection with pagination', ) async def read_all_records_paginated( - collection: str, - matching: str | None = None, - format: Format = Format.json, # noqa A002 - api_key: str = Depends(api_key_header_scheme), + collection: str, + matching: str | None = None, + format: Format = Format.json, # noqa: A002 + api_key: str = Depends(api_key_header_scheme), ) -> Page[dict | str]: result_list = await _read_all_records( collection=collection, @@ -493,7 +488,7 @@ async def read_records_of_type( collection: str, class_name: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ): return await _read_records_of_type( @@ -518,7 +513,7 @@ async def read_records_of_type_paginated( collection: str, class_name: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ) -> Page[dict | str]: result_list = await _read_records_of_type( @@ -533,13 +528,12 @@ async def read_records_of_type_paginated( async def _read_all_records( - collection: str, - matching: str | None = None, - format: Format = Format.json, # noqa A002 - api_key: str = Depends(api_key_header_scheme), - bound: int | None = None, + collection: str, + matching: str | None = None, + format: Format = Format.json, # noqa: A002 + api_key: str = Depends(api_key_header_scheme), + bound: int | None = None, ) -> LazyList: - def convert_to_http_exception(e: BaseException): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, @@ -591,7 +585,7 @@ async def _read_records_of_type( collection: str, class_name: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), bound: int | None = None, ) -> LazyList: @@ -622,7 +616,9 @@ async def _read_records_of_type( matching=matching, ) if bound: - check_bounds(len(token_store_list), bound, collection, f'/records/p/{class_name}') + check_bounds( + len(token_store_list), bound, collection, f'/records/p/{class_name}' + ) result_list.add_list(token_store_list) if final_permissions.curated_read: @@ -634,7 +630,12 @@ async def _read_records_of_type( matching=matching, ) if bound: - check_bounds(len(curated_store_list), bound, collection, f'/records/p/{class_name}') + check_bounds( + len(curated_store_list), + bound, + collection, + f'/records/p/{class_name}', + ) result_list.add_list(curated_store_list) # Sort the result list. @@ -664,7 +665,7 @@ async def _read_records_of_type( async def delete_record( collection: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): check_collection(g_configuration, collection) final_permissions, token_store = await process_token( @@ -682,8 +683,8 @@ async def delete_record( raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"Could not remove record with PID '{pid}' from the " - "token associated incoming area of collection " - f"'{collection}'.", + 'token associated incoming area of collection ' + f"'{collection}'.", ) return True diff --git a/dump_things_service/manifest.py b/dump_things_service/manifest.py index 71cb722..6d4e6de 100644 --- a/dump_things_service/manifest.py +++ b/dump_things_service/manifest.py @@ -12,7 +12,6 @@ from dump_things_service.collection import ( ) from dump_things_service.instance_state import InstanceState - logger = logging.getLogger('dump_things_service') tag_groups = [ @@ -58,10 +57,9 @@ openapi_tags_template = [ ] - def manifest_configuration( - configuration: Configuration, - instance_state: InstanceState, + configuration: Configuration, + instance_state: InstanceState, ): """Interpret the configuration and instantiate respective objects @@ -160,23 +158,23 @@ def manifest_configuration( def create_token( - instance_state: InstanceState, - token_name: str, - token_configuration: TokenConfig, + instance_state: InstanceState, + token_name: str, + token_configuration: TokenConfig, ): instance_state.tokens[token_name] = token_configuration def delete_token( - instance_state: InstanceState, - token_name: str, + instance_state: InstanceState, + token_name: str, ): instance_state.tokens.pop(token_name) def delete_collection( - instance_state: InstanceState, - collection_name: str, + instance_state: InstanceState, + collection_name: str, ): instance_state.collections.pop(collection_name) @@ -190,7 +188,6 @@ def create_openapi_tags( instance_state: InstanceState, openapi_tags_template: list[dict | str], ) -> list[dict]: - # Collect tag name lists for all tag groups that we have defined. tag_group_info = { tag_group: sorted( @@ -198,12 +195,12 @@ def create_openapi_tags( {'name': collection_info.tag_info[tag_group]} for collection_info in instance_state.collections.values() ], - key=lambda x: x['name'] + key=lambda x: x['name'], ) for tag_group in tag_groups } result = openapi_tags_template.copy() for tag_group, tag_list in tag_group_info.items(): index = result.index(tag_group) - result[index:index + 1] = tag_list + result[index : index + 1] = tag_list return result diff --git a/dump_things_service/model.py b/dump_things_service/model.py index 9155490..94f0daf 100644 --- a/dump_things_service/model.py +++ b/dump_things_service/model.py @@ -1,6 +1,6 @@ from __future__ import annotations -import dataclasses # noqa F401 -- used by generated code +import dataclasses # noqa: F401 -- used by generated code import logging import sys from functools import cache @@ -11,9 +11,9 @@ from typing import ( ) from urllib.parse import urlparse -import annotated_types # noqa F401 -- used by generated code -import pydantic # noqa F401 -- used by generated code -import pydantic_core # noqa F401 -- used by generated code +import annotated_types # noqa: F401 -- used by generated code +import pydantic # noqa: F401 -- used by generated code +import pydantic_core # noqa: F401 -- used by generated code from linkml.generators import ( PydanticGenerator, PythonGenerator, @@ -22,7 +22,7 @@ from linkml_runtime import SchemaView from pydantic._internal._model_construction import ModelMetaclass # Ensure linkml is patched -import dump_things_service.patches.enabled # noqa F401 -- apply patches +import dump_things_service.patches.enabled # noqa: F401 -- apply patches if TYPE_CHECKING: from types import ModuleType @@ -67,11 +67,11 @@ def get_subclasses( # TODO: shall we use the following code? # The code below would use schema-definitions to determine classes and not -# go through thw pydantic module generation. +# go through the pydantic module generation. @cache def get_subclasses_2( - collection_name: str, - class_name: str, + collection_name: str, + class_name: str, ) -> list[str]: from dump_things_service.instance_state import get_instance_state @@ -88,8 +88,8 @@ def compile_module_with_increasing_recursion_limit( module = None module_name = ( - urlparse(schema_location).path - .replace('/', '_') + urlparse(schema_location) + .path.replace('/', '_') .replace('-', '_') .replace('.', '_') ) diff --git a/dump_things_service/store/model_store.py b/dump_things_service/store/model_store.py index f77be4e..61600a5 100644 --- a/dump_things_service/store/model_store.py +++ b/dump_things_service/store/model_store.py @@ -17,8 +17,8 @@ if TYPE_CHECKING: from pydantic import BaseModel from dump_things_service.backends import ( - _RecordInfo, StorageBackend, + _RecordInfo, ) from dump_things_service.lazy_list import LazyList @@ -28,12 +28,7 @@ submitter_namespace = 'http://purl.obolibrary.org/obo/' class _ModelStore: - def __init__( - self, - schema: str, - backend: StorageBackend, - tags: dict[str, str] - ): + def __init__(self, schema: str, backend: StorageBackend, tags: dict[str, str]): self.schema = schema self.model = get_model_for_schema(self.schema)[0] self.backend = backend @@ -43,11 +38,13 @@ class _ModelStore: return self.backend.get_uri() def store_object( - self, - obj: BaseModel, - submitter: str | None, + self, + obj: BaseModel, + submitter: str | None, ) -> Iterable[tuple[str, dict]]: - if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in (obj.annotations or dict()): + if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in ( + obj.annotations or {} + ): return [] # Extract inlined records from the object, store individual records @@ -64,15 +61,15 @@ class _ModelStore: ] def pid_to_iri( - self, - pid: str, + self, + pid: str, ): return resolve_curie(self.model, pid) def _store_flat_object( - self, - obj: BaseModel, - submitter: str | None, + self, + obj: BaseModel, + submitter: str | None, ) -> dict: iri = self.pid_to_iri(obj.pid) class_name = obj.__class__.__name__ @@ -94,9 +91,9 @@ class _ModelStore: return json_object def annotate( - self, - json_object: dict, - submitter: str, + self, + json_object: dict, + submitter: str, ) -> None: """Add submitter IRI to the record annotations, use CURIE if possible""" json_object['annotations'] = self.homogenize_annotations(json_object) @@ -113,8 +110,8 @@ class _ModelStore: } def get_curie( - self, - curie_or_iri: str, + self, + curie_or_iri: str, ) -> str: if is_curie(curie_or_iri): return curie_or_iri @@ -131,8 +128,8 @@ class _ModelStore: return curie_or_iri def extract_inlined( - self, - record: BaseModel, + self, + record: BaseModel, ) -> list[BaseModel]: # The trivial case: no relations if not hasattr(record, 'relations') or record.relations is None: @@ -146,7 +143,8 @@ class _ModelStore: # Do not extract 'empty'-Thing records with an # `dlthings:placeholder` annotation. These records are just # placeholders for already extracted records. - if sub_record != self.model.Thing( + if sub_record + != self.model.Thing( pid=sub_record.pid, annotations={ 'dlthings:placeholder': sub_record.pid, @@ -165,21 +163,21 @@ class _ModelStore: pid=sub_record_pid, annotations={ 'dlthings:placeholder': sub_record_pid, - } + }, ) for sub_record_pid in record.relations } return [new_record, *extracted_sub_records] def get_object_by_pid( - self, - pid: str, + self, + pid: str, ) -> tuple[str, dict] | tuple[None, None]: return self.get_object_by_iri(self.pid_to_iri(pid)) def get_object_by_iri( - self, - iri: str, + self, + iri: str, ) -> tuple[str, dict] | tuple[None, None]: record_info = self.backend.get_record_by_iri(iri) if record_info: @@ -187,11 +185,11 @@ class _ModelStore: return None, None def get_objects_of_class( - self, - class_name: str, - matching: str | None, - *, - include_subclasses: bool = True, + self, + class_name: str, + matching: str | None, + *, + include_subclasses: bool = True, ) -> LazyList[_RecordInfo]: """ Get all objects of a specific class. @@ -210,8 +208,8 @@ class _ModelStore: return self.backend.get_records_of_classes(class_names, matching) def get_all_objects( - self, - matching: str | None = None, + self, + matching: str | None = None, ) -> LazyList[_RecordInfo]: """ Get all objects of a specific class. @@ -222,8 +220,8 @@ class _ModelStore: return self.backend.get_all_records(matching) def delete_object( - self, - pid: str, + self, + pid: str, ) -> bool: return self.backend.remove_record(self.pid_to_iri(pid)) @@ -232,9 +230,9 @@ _existing_model_stores = {} def ModelStore( # noqa: N802 - schema: str, - backend: StorageBackend, - tags: dict[str, str], + schema: str, + backend: StorageBackend, + tags: dict[str, str], ) -> _ModelStore: """Create a unique model store for the given schema and backend. @@ -252,10 +250,9 @@ def ModelStore( # noqa: N802 # We store a pointer to the backend in the value to ensure that the # backend object exists while we use its `id` as a key. _existing_model_stores[id(backend)] = existing_model_store, backend - else: - # Check that the schemas are compatible, if the backend is reused. - if existing_model_store.schema != schema: - msg = 'Backend is already used in a ModelStore with a different schema' - raise ValueError(msg) + # Check that the schemas are compatible, if the backend is reused. + elif existing_model_store.schema != schema: + msg = 'Backend is already used in a ModelStore with a different schema' + raise ValueError(msg) return existing_model_store diff --git a/dump_things_service/tests/create_store.py b/dump_things_service/tests/create_store.py index 5f038c0..f5e0660 100644 --- a/dump_things_service/tests/create_store.py +++ b/dump_things_service/tests/create_store.py @@ -4,18 +4,19 @@ from typing import TYPE_CHECKING import yaml -from dump_things_service.backends.record_dir import RecordDirStore -from dump_things_service.backends.sqlite import ( - SQLiteBackend, - record_file_name as sqlite_record_file_name, -) from dump_things_service.abstract_config import ( - RecordDirBackendConfig, CollectionConfig, Configuration, MappingMethod, + RecordDirBackendConfig, mapping_functions, ) +from dump_things_service.backends.sqlite import ( + SQLiteBackend, +) +from dump_things_service.backends.sqlite import ( + record_file_name as sqlite_record_file_name, +) from dump_things_service.model import get_model_for_schema from dump_things_service.resolve_curie import resolve_curie diff --git a/dump_things_service/tests/fixtures.py b/dump_things_service/tests/fixtures.py index 1f19f86..e48cc79 100644 --- a/dump_things_service/tests/fixtures.py +++ b/dump_things_service/tests/fixtures.py @@ -11,20 +11,23 @@ import yaml from dump_things_service.abstract_config import ( GitAuditBackendConfig, SQLiteBackendConfig, + TagSpec, TokenCollectionConfig, - TokenModes, hash_token_representation, TagSpec, + TokenModes, + hash_token_representation, ) from dump_things_service.backends import StorageBackend from dump_things_service.backends.record_dir import RecordDirStore from dump_things_service.backends.sqlite import ( SQLiteBackend, +) +from dump_things_service.backends.sqlite import ( record_file_name as sqlite_db_filename, ) from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.instance_state import get_mapping_function_by_name from dump_things_service.model import get_model_for_schema from dump_things_service.resolve_curie import resolve_curie -from dump_things_service.token_endpoints import TokenRequest from dump_things_service.tests.create_store import ( pid, pid_curated, @@ -33,7 +36,7 @@ from dump_things_service.tests.create_store import ( test_record_curated, test_record_trr, ) - +from dump_things_service.token_endpoints import TokenRequest # String representation of curated- and incoming-path curated = 'curated' @@ -41,7 +44,9 @@ incoming = 'incoming' # Path to a local simple test schema test_schema_location = str((Path(__file__).parent / 'testschema.yaml').absolute()) -flat_social_schema_location = 'https://concepts.datalad.org/s/flat-social/unreleased.yaml' +flat_social_schema_location = ( + 'https://concepts.datalad.org/s/flat-social/unreleased.yaml' +) # The test store is created empty and collections are added via the admin @@ -64,7 +69,7 @@ g_default_collections[6].submission_tags = TagSpec( g_default_collections.append( CollectionRequest( - name=f'collection_8', + name='collection_8', default_token='test_default_token', curated=PurePosixPath(f'{curated}/collection_8'), schema=test_schema_location, @@ -75,38 +80,40 @@ g_default_collections.append( submission_tags=TagSpec( submitter_id_tag='no_default_id_tag', submission_time_tag='no_default_time_tag', - ) + ), ) ) -g_default_collections.extend([ - CollectionRequest( - name='collection_dlflatsocial-1', - schema=flat_social_schema_location, - default_token='test_default_token', - curated=PurePosixPath(f'{curated}/collection_dlflatsocial-1'), - incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-1'), - ), - CollectionRequest( - name='collection_dlflatsocial-2', - schema=flat_social_schema_location, - default_token='test_default_token', - curated=PurePosixPath(f'{curated}/collection_dlflatsocial-2'), - incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-2'), - backend=SQLiteBackendConfig( - type='sqlite', +g_default_collections.extend( + [ + CollectionRequest( + name='collection_dlflatsocial-1', + schema=flat_social_schema_location, + default_token='test_default_token', + curated=PurePosixPath(f'{curated}/collection_dlflatsocial-1'), + incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-1'), ), - use_classes=[ - 'Organization', - 'Person', - 'Project', - ], - ignore_classes=[ - 'Organization', - 'Project', - ], - ), -]) + CollectionRequest( + name='collection_dlflatsocial-2', + schema=flat_social_schema_location, + default_token='test_default_token', + curated=PurePosixPath(f'{curated}/collection_dlflatsocial-2'), + incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-2'), + backend=SQLiteBackendConfig( + type='sqlite', + ), + use_classes=[ + 'Organization', + 'Person', + 'Project', + ], + ignore_classes=[ + 'Organization', + 'Project', + ], + ), + ] +) g_default_tokens = [ TokenRequest( @@ -152,7 +159,7 @@ g_default_tokens = [ hashed=False, representation='token-2', collections={ - f'collection_2': TokenCollectionConfig( + 'collection_2': TokenCollectionConfig( mode=TokenModes.WRITE_COLLECTION, incoming_label='in_token-2', ) @@ -164,7 +171,7 @@ g_default_tokens = [ hashed=False, representation='token-8', collections={ - f'collection_8': TokenCollectionConfig( + 'collection_8': TokenCollectionConfig( mode=TokenModes.WRITE_COLLECTION, incoming_label='test_user_8', ) @@ -235,7 +242,7 @@ g_default_tokens = [ mode=TokenModes.WRITE_COLLECTION, incoming_label='modes', ), - } + }, ), TokenRequest( name='Test 0X000 (READ_SUBMISSIONS)', @@ -354,7 +361,8 @@ def fastapi_app_simple(dump_stores_simple): old_sys_argv = sys.argv sys.argv = [ 'test-runner', - '--admin-token-hash', hash_token_representation(admin_token), + '--admin-token-hash', + hash_token_representation(admin_token), '--ignore-default-config-file', str(tmp_path), ] @@ -429,15 +437,15 @@ def fastapi_client_simple(fastapi_app_simple): def add_records_to_backend( - backend: StorageBackend, - pydantic_module: ModuleType, - record_infos: list[tuple[str, str, str]], + backend: StorageBackend, + pydantic_module: ModuleType, + record_infos: list[tuple[str, str, str]], ): for class_name, record_pid, yaml_stream in record_infos: - json_object = yaml.load(yaml_stream, Loader=yaml.SafeLoader ) + json_object = yaml.load(yaml_stream, Loader=yaml.SafeLoader) assert record_pid == json_object['pid'] backend.add_record( iri=resolve_curie(pydantic_module, json_object['pid']), class_name=class_name, json_object=json_object, - ) + ) diff --git a/dump_things_service/tests/test_auth.py b/dump_things_service/tests/test_auth.py index efba9be..f970d73 100644 --- a/dump_things_service/tests/test_auth.py +++ b/dump_things_service/tests/test_auth.py @@ -15,11 +15,7 @@ user_1 = { '@type': 'user', } -org_1 = { - 'id': 1, - 'name': 'org_1', - '@type': 'org' -} +org_1 = {'id': 1, 'name': 'org_1', '@type': 'org'} repo_1 = { 'id': 3, @@ -46,10 +42,18 @@ team_3 = json.loads(team_template.format(id=3, action='write')) def setup_http_server(http_server) -> None: for instance in ('1', '2'): http_server.expect_request(f'/api/v{instance}/user').respond_with_json(user_1) - http_server.expect_request(f'/api/v{instance}/user/teams').respond_with_json([team_1, team_3]) - http_server.expect_request(f'/api/v{instance}/orgs/org_1').respond_with_json(org_1) - http_server.expect_request(f'/api/v{instance}/orgs/org_1/teams').respond_with_json([team_1, team_2, team_3]) - http_server.expect_request(f'/api/v{instance}/repos/org_1/repo_1/teams').respond_with_json([team_1, team_2, team_3]) + http_server.expect_request(f'/api/v{instance}/user/teams').respond_with_json( + [team_1, team_3] + ) + http_server.expect_request(f'/api/v{instance}/orgs/org_1').respond_with_json( + org_1 + ) + http_server.expect_request( + f'/api/v{instance}/orgs/org_1/teams' + ).respond_with_json([team_1, team_2, team_3]) + http_server.expect_request( + f'/api/v{instance}/repos/org_1/repo_1/teams' + ).respond_with_json([team_1, team_2, team_3]) @pytest.mark.parametrize('repository', ['repo_1', None]) diff --git a/dump_things_service/tests/test_basic.py b/dump_things_service/tests/test_basic.py index 7591678..5e88c02 100644 --- a/dump_things_service/tests/test_basic.py +++ b/dump_things_service/tests/test_basic.py @@ -1,7 +1,3 @@ - -import pytest # F401 - -from . import schema_file from .. import ( HTTP_200_OK, HTTP_400_BAD_REQUEST, @@ -11,14 +7,13 @@ from .. import ( HTTP_503_SERVICE_UNAVAILABLE, ) from ..__about__ import __version__ -from ..utils import cleaned_json +from . import schema_file from .create_store import ( given_name, pid, ) from .test_utils import basic_write_locations - extra_record = { 'schema_type': 'abc:Person', 'pid': 'abc:aaaa', @@ -298,7 +293,7 @@ def test_funky_pid(fastapi_client_simple): def test_token_store_priority(fastapi_client_simple): - test_client, store_dir, _ = fastapi_client_simple + test_client, _store_dir, _ = fastapi_client_simple # Post a record with the same pid as the global store's test record, but # with different content. @@ -393,7 +388,8 @@ def test_server(fastapi_client_simple): 'classes': test_schema_classes, } for i in range(1, 9) - ] + [ + ] + + [ { 'name': f'collection_dlflatsocial-{i}', 'schema': 'https://concepts.datalad.org/s/flat-social/unreleased.yaml', diff --git a/dump_things_service/tests/test_collection_administration.py b/dump_things_service/tests/test_collection_administration.py index 5dc2f39..6e9a943 100644 --- a/dump_things_service/tests/test_collection_administration.py +++ b/dump_things_service/tests/test_collection_administration.py @@ -6,10 +6,10 @@ from pathlib import ( from starlette.testclient import TestClient from dump_things_service import ( - HTTP_201_CREATED, HTTP_200_OK, - HTTP_404_NOT_FOUND, + HTTP_201_CREATED, HTTP_401_UNAUTHORIZED, + HTTP_404_NOT_FOUND, ) from dump_things_service.abstract_config import ( GitAuditBackendConfig, @@ -19,10 +19,9 @@ from dump_things_service.abstract_config import ( ) from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.token_endpoints import ( - TokenRequest, AdminTokenRequest, + TokenRequest, ) -from dump_things_service.utils import cleaned_json # String representation of curated- and incoming-path curated = 'admin_test_curated' @@ -55,7 +54,7 @@ new_token_request = TokenRequest( }, ) -new_admin_token_name='New_Admin_Token' +new_admin_token_name = 'New_Admin_Token' plain_new_admin_token = 'admin-XXX' new_admin_token_request = AdminTokenRequest( name=new_admin_token_name, @@ -64,15 +63,12 @@ new_admin_token_request = AdminTokenRequest( def _name_in_openapi_paths( - test_client: TestClient, - name: str, + test_client: TestClient, + name: str, ) -> bool: response = test_client.get('/openapi.json') open_api = response.json() - for path in open_api['paths'].keys(): - if name in path: - return True - return False + return any(name in path for path in open_api['paths']) def test_collection_adding(fastapi_client_simple): @@ -100,7 +96,9 @@ def test_collection_adding(fastapi_client_simple): headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == HTTP_200_OK - new_collection_config = new_collection_request.model_dump(mode='json', by_alias=True) + new_collection_config = new_collection_request.model_dump( + mode='json', by_alias=True + ) del new_collection_config['name'] assert response.json() == new_collection_config @@ -123,7 +121,7 @@ def test_collection_adding(fastapi_client_simple): 'user_id': new_token_request.user_id, 'collections': new_token_request.model_dump(mode='json')['collections'], 'hashed': new_token_request.hashed, - 'representation': new_token_request.representation + 'representation': new_token_request.representation, } new_record = { @@ -204,7 +202,7 @@ def test_collection_putting(fastapi_client_simple, tmp_path): path=Path(tmp_path), auto_flush_timeout=2, ) - ] + ], ) # Check that the collection does not yet exist @@ -259,7 +257,7 @@ def test_collection_reading(fastapi_client_simple): # Check that the new admin token is not yet working response = test_client.get( - f'/collections', + '/collections', headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == HTTP_200_OK @@ -273,7 +271,7 @@ def test_admin_token_management(fastapi_client_simple): # Check that the new admin token is not yet working response = test_client.get( - f'/collections/collection_1', + '/collections/collection_1', headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_401_UNAUTHORIZED @@ -288,14 +286,14 @@ def test_admin_token_management(fastapi_client_simple): # Try the new token response = test_client.get( - f'/collections/collection_1', + '/collections/collection_1', headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_200_OK # Check that the token shows up in the token list response = test_client.get( - f'/admin_tokens', + '/admin_tokens', headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_200_OK @@ -310,7 +308,7 @@ def test_admin_token_management(fastapi_client_simple): assert response.status_code == HTTP_200_OK response = test_client.get( - f'/admin_tokens', + '/admin_tokens', headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == HTTP_200_OK diff --git a/dump_things_service/tests/test_config.py b/dump_things_service/tests/test_config.py index 72efe9c..1d2c30d 100644 --- a/dump_things_service/tests/test_config.py +++ b/dump_things_service/tests/test_config.py @@ -22,13 +22,12 @@ from dump_things_service.exceptions import ConfigError from dump_things_service.tests import schema_file from dump_things_service.token_endpoints import TokenRequest - collection_request_pattern = CollectionRequest( name='', schema=str(schema_file), default_token='test_default_token', curated=PurePosixPath('curate_dir'), - incoming=PurePosixPath(f'incoming_dir'), + incoming=PurePosixPath('incoming_dir'), ) @@ -36,13 +35,13 @@ def test_illegal_collection_name_detection(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple for name in ( - 'collections', - 'tokens', - 'admin_tokens', - dump_things_private_collection_name, + 'collections', + 'tokens', + 'admin_tokens', + dump_things_private_collection_name, ): response = test_client.post( - f'/collections', + '/collections', json={ **collection_request_pattern.model_dump(mode='json', by_alias=True), 'name': name, @@ -52,17 +51,19 @@ def test_illegal_collection_name_detection(fastapi_client_simple): assert response.status_code == HTTP_409_CONFLICT -@pytest.mark.skip(reason='Reuse detection is disabled to support existing old configurations') +@pytest.mark.skip( + reason='Reuse detection is disabled to support existing old configurations' +) def test_collection_dir_reuse_detection(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple for curated_path, incoming_path in ( - ('curated/collection_1', 'incoming/XXXX'), - ('curated/XXXX', 'incoming/collection_1'), - ('curated/collection_1', 'incoming/collection_2'), + ('curated/collection_1', 'incoming/XXXX'), + ('curated/XXXX', 'incoming/collection_1'), + ('curated/collection_1', 'incoming/collection_2'), ): response = test_client.post( - f'/collections', + '/collections', json={ **collection_request_pattern.model_dump(mode='json', by_alias=True), 'curated': curated_path, @@ -76,15 +77,17 @@ def test_collection_dir_reuse_detection(fastapi_client_simple): def test_scanner_error_detection(tmp_path_factory): tmp_path = tmp_path_factory.mktemp('config_scanner_test') - config_backend, audit_backend = get_config_backends(tmp_path) + config_backend, _audit_backend = get_config_backends(tmp_path) config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object={'pid': dump_things_config_iri} + json_object={'pid': dump_things_config_iri}, ) md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest() - config_file_path = config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + config_file_path = ( + config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + ) config_file_path.write_text('collections: ::: -\n sdsdfsdf: xxx') with pytest.raises(ConfigError): read_config(tmp_path, force_reload=True) @@ -93,15 +96,17 @@ def test_scanner_error_detection(tmp_path_factory): def test_structure_error_detection(tmp_path_factory): tmp_path = tmp_path_factory.mktemp('config_scanner_test') - config_backend, audit_backend = get_config_backends(tmp_path) + config_backend, _audit_backend = get_config_backends(tmp_path) config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object={'pid': dump_things_config_iri} + json_object={'pid': dump_things_config_iri}, ) md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest() - config_file_path = config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + config_file_path = ( + config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + ) config_file_path.write_text('type: 1\n') with pytest.raises(ConfigError): read_config(tmp_path, force_reload=True) @@ -135,7 +140,7 @@ def test_missing_incoming_detection(fastapi_client_simple): mode=TokenModes.CURATOR, incoming_label='', ) - } + }, ) # Check that a write token for a collection without incoming path cannot @@ -155,7 +160,9 @@ def test_missing_incoming_detection(fastapi_client_simple): assert response.status_code == HTTP_200_OK # Add a collection with incoming path - collection_request.incoming = PurePosixPath('missing_incoming_detection_test_incoming') + collection_request.incoming = PurePosixPath( + 'missing_incoming_detection_test_incoming' + ) response = test_client.post( '/collections', json=collection_request.model_dump(mode='json', by_alias=True), @@ -173,9 +180,11 @@ def test_missing_incoming_detection(fastapi_client_simple): assert response.status_code == HTTP_406_NOT_ACCEPTABLE # Check that a write token for a collection with an incoming path can be created - token_request.collections['missing_incoming_detection_test'] = TokenCollectionConfig( - mode=TokenModes.CURATOR, - incoming_label='test_incoming_label', + token_request.collections['missing_incoming_detection_test'] = ( + TokenCollectionConfig( + mode=TokenModes.CURATOR, + incoming_label='test_incoming_label', + ) ) response = test_client.post( '/tokens', diff --git a/dump_things_service/tests/test_curated.py b/dump_things_service/tests/test_curated.py index 47737a8..7e121ec 100644 --- a/dump_things_service/tests/test_curated.py +++ b/dump_things_service/tests/test_curated.py @@ -1,17 +1,17 @@ from __future__ import annotations -import pytest import time -import yaml from itertools import count +import pytest +import yaml + from dump_things_service import ( HTTP_200_OK, HTTP_404_NOT_FOUND, ) from dump_things_service.instance_state import get_instance_state - delete_record = { 'schema_type': 'abc:Person', 'pid': 'abc:delete-me', @@ -19,8 +19,8 @@ delete_record = { } -@pytest.mark.parametrize('paginate', ('', 'p/')) -@pytest.mark.parametrize('class_name', ('', 'Person')) +@pytest.mark.parametrize('paginate', ['', 'p/']) +@pytest.mark.parametrize('class_name', ['', 'Person']) def test_read_curated_records( fastapi_client_simple, paginate, @@ -54,10 +54,6 @@ def test_read_curated_records( assert len(json_object) == count -pytest.mark.parametrize( - 'pid', - ('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'), -) def test_read_curated_records_by_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple @@ -185,5 +181,6 @@ def test_audit_backend_auto_flush(fastapi_client_simple): break i += 1 if i == 10: - raise ValueError(f'auto flush did not trigger within 10 seconds') + msg = 'auto flush did not trigger within 10 seconds' + raise ValueError(msg) time.sleep(1) diff --git a/dump_things_service/tests/test_extract_inline.py b/dump_things_service/tests/test_extract_inline.py index 02c7440..5501174 100644 --- a/dump_things_service/tests/test_extract_inline.py +++ b/dump_things_service/tests/test_extract_inline.py @@ -113,7 +113,10 @@ empty_inlined_json_record = cleaned_json(dataclasses.asdict(empty_inlined_object tree = ( - ('dlflatsocial:test_extract_1', ('dlflatsocial:test_extract_1_1', 'dlflatsocial:test_extract_1_2')), + ( + 'dlflatsocial:test_extract_1', + ('dlflatsocial:test_extract_1_1', 'dlflatsocial:test_extract_1_2'), + ), ('dlflatsocial:test_extract_1_1', ('dlflatsocial:test_extract_1_1_1',)), ('dlflatsocial:test_extract_1_2', ()), ('dlflatsocial:test_extract_1_1_1', ()), @@ -181,10 +184,10 @@ def test_inline_extraction_locally(): store = ModelStore( schema=str(schema_path), backend=None, - tags = { + tags={ 'id': 'abc:id', 'time': 'abc:time', - } + }, ) store.model = MockedModule() records = store.extract_inlined(inlined_object) @@ -216,7 +219,7 @@ def test_dont_extract_empty_things_locally(): tags={ 'id': 'https://id', 'time': 'https://time', - } + }, ) store.model = MockedModule() records = store.extract_inlined(empty_inlined_object) @@ -257,7 +260,10 @@ def test_inline_extraction_on_service(fastapi_client_simple): # Check that individual record classes were recognized for class_name, pids in ( - ('Person', ('dlflatsocial:test_extract_1', 'dlflatsocial:test_extract_1_1')), + ( + 'Person', + ('dlflatsocial:test_extract_1', 'dlflatsocial:test_extract_1_1'), + ), ('Agent', ('dlflatsocial:test_extract_1_1_1',)), ('InstantaneousEvent', ('dlflatsocial:test_extract_1_2',)), ): @@ -301,7 +307,10 @@ def test_inline_ttl_processing(fastapi_client_simple): # Check that individual record classes were recognized for class_name, pids in ( - ('Person', ('dlflatsocial:test_ttl_inline_1', 'dlflatsocial:test_ttl_inline_1_1')), + ( + 'Person', + ('dlflatsocial:test_ttl_inline_1', 'dlflatsocial:test_ttl_inline_1_1'), + ), ('Agent', ('dlflatsocial:test_ttl_inline_1_1_1',)), ('InstantaneousEvent', ('dlflatsocial:test_ttl_inline_1_2',)), ): @@ -339,7 +348,7 @@ def _check_result_json( # That breaks the tests. They assume that Person.relations has range Thing. @pytest.mark.xfail def test_dont_extract_empty_things_on_service(fastapi_client_simple): - test_client, store = fastapi_client_simple + test_client, _store = fastapi_client_simple for i in range(1, 3): # Deposit JSON record @@ -352,7 +361,7 @@ def test_dont_extract_empty_things_on_service(fastapi_client_simple): def test_store_things(fastapi_client_simple): - test_client, store, _ = fastapi_client_simple + test_client, _store, _ = fastapi_client_simple simple_thing = { 'pid': 'http://test.simple.thing/1', @@ -375,7 +384,7 @@ def test_store_things(fastapi_client_simple): def test_store_complex_things(fastapi_client_simple): - test_client, store, _ = fastapi_client_simple + test_client, _store, _ = fastapi_client_simple complex_thing = { 'pid': 'http://test.complex.thing/1', @@ -386,9 +395,9 @@ def test_store_complex_things(fastapi_client_simple): 'http://test.complex.thing/1.1.1': { 'pid': 'http://test.complex.thing/1.1.1', } - } + }, } - } + }, } # Deposit JSON record @@ -402,9 +411,9 @@ def test_store_complex_things(fastapi_client_simple): # Try to read individual extracted elements for pid in ( - 'http://test.complex.thing/1', - 'http://test.complex.thing/1.1', - 'http://test.complex.thing/1.1.1', + 'http://test.complex.thing/1', + 'http://test.complex.thing/1.1', + 'http://test.complex.thing/1.1.1', ): response = test_client.get( f'/collection_1/record?pid={pid}', diff --git a/dump_things_service/tests/test_ifabsent_patch.py b/dump_things_service/tests/test_ifabsent_patch.py index 95e855b..e1f4312 100644 --- a/dump_things_service/tests/test_ifabsent_patch.py +++ b/dump_things_service/tests/test_ifabsent_patch.py @@ -6,7 +6,6 @@ import linkml.generators.common.ifabsent_processor as if_abs_proc import dump_things_service.patches.ifabsent_processing - # Path to a local simple test schema schema_dir = Path(__file__).parent / 'assets' @@ -17,10 +16,11 @@ def _original_uri_for(self, s: str) -> str: def test_ifabsent_patch(): - # Patch in the faulty, original code and check for its result if_abs_proc.IfAbsentProcessor._uri_for = _original_uri_for - gen1 = linkml.generators.PydanticGenerator(str(schema_dir / 'schema-ifabsent-error.yaml')) + gen1 = linkml.generators.PydanticGenerator( + str(schema_dir / 'schema-ifabsent-error.yaml') + ) x = gen1.serialize() assert 'default=XSD["04fa4r544"]' in x @@ -28,6 +28,8 @@ def test_ifabsent_patch(): reload(dump_things_service.patches.ifabsent_processing) # Check for proper code generation - gen2 = linkml.generators.PydanticGenerator(str(schema_dir / 'schema-ifabsent-error.yaml')) + gen2 = linkml.generators.PydanticGenerator( + str(schema_dir / 'schema-ifabsent-error.yaml') + ) y = gen2.serialize() assert 'XSD' not in y diff --git a/dump_things_service/tests/test_incoming.py b/dump_things_service/tests/test_incoming.py index 15b27a4..456cc69 100644 --- a/dump_things_service/tests/test_incoming.py +++ b/dump_things_service/tests/test_incoming.py @@ -30,6 +30,7 @@ def test_incoming_labels(fastapi_client_simple): zones_filled = False + def fill_zones(test_client): global zones_filled @@ -53,33 +54,33 @@ def fill_zones(test_client): json={ 'pid': f'abc:test_incoming-collection_{collection_id}-{token}', 'given_name': f'collection_{collection_id}-{token}', - } + }, ) assert result.status_code == HTTP_200_OK zones_filled = True -@pytest.mark.parametrize('paginate', ('', 'p/')) -@pytest.mark.parametrize('class_name', ('', 'Person')) +@pytest.mark.parametrize('paginate', ['', 'p/']) +@pytest.mark.parametrize('class_name', ['', 'Person']) def test_read_incoming_records( - fastapi_client_simple, - paginate: str, - class_name: str, + fastapi_client_simple, + paginate: str, + class_name: str, ): test_client, _, _ = fastapi_client_simple fill_zones(test_client) for collection_id, labels in ( - (1, ['modes', 'admin_1', 'in_token_1']), - (2, ['in_token-2', 'admin_2']), - (3, ['admin_3']), - (4, ['admin_4']), - (5, ['admin_common']), - (6, ['admin_common']), - (7, ['admin_common']), - (8, ['modes', 'test_user_8', 'admin_common']), + (1, ['modes', 'admin_1', 'in_token_1']), + (2, ['in_token-2', 'admin_2']), + (3, ['admin_3']), + (4, ['admin_4']), + (5, ['admin_common']), + (6, ['admin_common']), + (7, ['admin_common']), + (8, ['modes', 'test_user_8', 'admin_common']), ): # Check that all incoming zones are reached for label in labels: @@ -87,7 +88,9 @@ def test_read_incoming_records( f'/collection_{collection_id}/incoming/{label}/records/{paginate}{class_name}', headers={'x-dumpthings-token': 'token_curator'}, ) - assert response.status_code == HTTP_200_OK, f'failed on collection: {collection_id}, label: {label}, class: {class_name}' + assert response.status_code == HTTP_200_OK, ( + f'failed on collection: {collection_id}, label: {label}, class: {class_name}' + ) # We don't know the exact number of entries in each zone, because # it depends on the tests that ran before. @@ -103,22 +106,15 @@ def test_read_incoming_records( ) assert response.status_code == HTTP_200_OK json_object = response.json() - if 'items' in json_object: - result = json_object['items'] - else: - result = json_object + result = json_object['items'] if 'items' in json_object else json_object matching = [ - json_object - for json_object in result - if json_object['pid'] == pattern + json_object for json_object in result if json_object['pid'] == pattern ] - assert len(matching) == expected_length, f'did not find {expected_length} record: collection_{collection_id}, {label}, {result}' + assert len(matching) == expected_length, ( + f'did not find {expected_length} record: collection_{collection_id}, {label}, {result}' + ) -pytest.mark.parametrize( - 'pid', - ('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'), -) def test_read_incoming_records_by_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple diff --git a/dump_things_service/tests/test_modes.py b/dump_things_service/tests/test_modes.py index 2b22206..07afa32 100644 --- a/dump_things_service/tests/test_modes.py +++ b/dump_things_service/tests/test_modes.py @@ -50,7 +50,7 @@ def verify_modes( def test_token_modes(fastapi_client_simple): - test_client, store_dir, _ = fastapi_client_simple + test_client, _store_dir, _ = fastapi_client_simple # Post a record to incoming of collections `collection_1`. We use it to # validate read/write permissions on class-base diff --git a/dump_things_service/tests/test_roundtrip.py b/dump_things_service/tests/test_roundtrip.py index 50bbb7f..0b9608d 100644 --- a/dump_things_service/tests/test_roundtrip.py +++ b/dump_things_service/tests/test_roundtrip.py @@ -1,5 +1,5 @@ import freezegun -import pytest # noqa F401 +import pytest # noqa: F401 from .. import HTTP_200_OK from ..utils import cleaned_json diff --git a/dump_things_service/tests/test_roundtrip_flatsocial.py b/dump_things_service/tests/test_roundtrip_flatsocial.py index bb2481c..f1e1797 100644 --- a/dump_things_service/tests/test_roundtrip_flatsocial.py +++ b/dump_things_service/tests/test_roundtrip_flatsocial.py @@ -1,6 +1,5 @@ -import pytest # noqa F401 - import freezegun +import pytest # noqa: F401 from .. import HTTP_200_OK from ..utils import cleaned_json @@ -144,7 +143,9 @@ def test_ttl_json_ttl_dlflatsocial(fastapi_client_simple): }, data=ttl_input_record, ) - assert response.status_code == HTTP_200_OK, 'Response content: ' + response.content.decode() + assert response.status_code == HTTP_200_OK, ( + 'Response content: ' + response.content.decode() + ) # Retrieve JSON records response = test_client.get( @@ -172,8 +173,12 @@ def test_ttl_json_ttl_dlflatsocial(fastapi_client_simple): assert response.status_code == HTTP_200_OK assert ( response.text.strip() - == ttl_output_record_a.replace('dlflatsocial:test_john_ttl', new_json_pid).strip() + == ttl_output_record_a.replace( + 'dlflatsocial:test_john_ttl', new_json_pid + ).strip() ) or ( response.text.strip() - == ttl_output_record_b.replace('dlflatsocial:test_john_ttl', new_json_pid).strip() + == ttl_output_record_b.replace( + 'dlflatsocial:test_john_ttl', new_json_pid + ).strip() ) diff --git a/dump_things_service/tests/test_token_endpoints.py b/dump_things_service/tests/test_token_endpoints.py index 0209eb3..0af5019 100644 --- a/dump_things_service/tests/test_token_endpoints.py +++ b/dump_things_service/tests/test_token_endpoints.py @@ -11,14 +11,11 @@ def test_token_creation(fastapi_client_simple): 'user_id': 'u_a', 'representation': '8bb6805ff10bcb1c2ca49dcd4bfef94d', 'collections': { - 'collection_1': { - 'mode': 'WRITE_COLLECTION', - 'incoming_label': 'i_a' - } - } + 'collection_1': {'mode': 'WRITE_COLLECTION', 'incoming_label': 'i_a'} + }, } - # Create a token eith name 'a' + # Create a token with name 'a' response = test_client.post( '/tokens', headers={'x-dumpthings-token': admin_token}, @@ -34,7 +31,7 @@ def test_token_creation(fastapi_client_simple): ) assert response.status_code == HTTP_409_CONFLICT - # Try to create another token eith name 'b' and the same representation + # Try to create another token with name 'b' and the same representation # as 'a', should result in a 4ß9-error json_record['name'] = 'b' response = test_client.post( diff --git a/dump_things_service/tests/test_unicode.py b/dump_things_service/tests/test_unicode.py index 15cf7cc..e26e313 100644 --- a/dump_things_service/tests/test_unicode.py +++ b/dump_things_service/tests/test_unicode.py @@ -2,7 +2,6 @@ from pathlib import Path from .. import HTTP_200_OK - # Path to a local simple test schema schema_file = Path(__file__).parent / 'testschema.yaml' @@ -31,9 +30,9 @@ def test_unicode_iri(fastapi_client_simple): response = test_client.post( '/collection_1/record/Person', headers={'x-dumpthings-token': 'token-1'}, - json = { + json={ 'pid': 'https://en.wikipedia.org/wiki/Universita_degli_Studi_eCampus', - 'given_name': 'Università degli Studi eCampus (Italy)', - } + 'given_name': 'Università degli Studi eCampus (Italy)', # codespell:ignore + }, ) assert response.status_code == HTTP_200_OK diff --git a/dump_things_service/tests/test_validate.py b/dump_things_service/tests/test_validate.py index 800b17c..d6c3e10 100644 --- a/dump_things_service/tests/test_validate.py +++ b/dump_things_service/tests/test_validate.py @@ -1,9 +1,11 @@ - from dump_things_service import HTTP_422_UNPROCESSABLE_CONTENT json_records = [ ({'name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT), - ({'given_name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT), + ( + {'given_name': 'Henry', 'pid': 'unknown_prefix:henry'}, + HTTP_422_UNPROCESSABLE_CONTENT, + ), ({'given_name': 'Henry', 'pid': 'xyz:henry'}, 200), ] diff --git a/dump_things_service/tests/test_web_interface.py b/dump_things_service/tests/test_web_interface.py index 8096d5a..74464e2 100644 --- a/dump_things_service/tests/test_web_interface.py +++ b/dump_things_service/tests/test_web_interface.py @@ -14,15 +14,15 @@ pids = ('', '--------', '&&&&&', 'abc', 'abc&', 'abc&format=ttl') @pytest.mark.parametrize( - 'collection_name,class_name,query,format_name', # noqa PT006 + 'collection_name,class_name,query,format_name', # noqa: PT006 tuple(product(*(collection_names, class_names, queries, format_names))), ) def test_web_interface_post_errors( - fastapi_client_simple, - collection_name, - class_name, - query, - format_name, + fastapi_client_simple, + collection_name, + class_name, + query, + format_name, ): """Check that no internal server error occurs with weird input""" test_client, _, _ = fastapi_client_simple @@ -35,15 +35,15 @@ def test_web_interface_post_errors( @pytest.mark.parametrize( - 'collection_name,class_name,query,format_name', # noqa PT006 + 'collection_name,class_name,query,format_name', # noqa: PT006 tuple(product(*(collection_names, class_names, queries, format_names))), ) def test_web_interface_get_class_errors( - fastapi_client_simple, - collection_name, - class_name, - query, - format_name, + fastapi_client_simple, + collection_name, + class_name, + query, + format_name, ): """Check that no internal server error occurs with weird input""" test_client, _, _ = fastapi_client_simple @@ -60,15 +60,15 @@ def test_web_interface_get_class_errors( @pytest.mark.parametrize( - 'collection_name,pid,query,format_name', # noqa PT006 + 'collection_name,pid,query,format_name', # noqa: PT006 tuple(product(*(collection_names, pids, queries, format_names))), ) def test_web_interface_get_pid_errors( - fastapi_client_simple, - collection_name, - pid, - query, - format_name, + fastapi_client_simple, + collection_name, + pid, + query, + format_name, ): """Check that no internal server error occurs with weird input""" test_client, _, _ = fastapi_client_simple diff --git a/dump_things_service/token_endpoints.py b/dump_things_service/token_endpoints.py index 06d2d0f..c7e6b0f 100644 --- a/dump_things_service/token_endpoints.py +++ b/dump_things_service/token_endpoints.py @@ -1,6 +1,7 @@ import logging import random import re +from typing import Annotated from urllib.parse import quote from fastapi import ( @@ -30,12 +31,11 @@ from dump_things_service.abstract_config import ( ) from dump_things_service.admin import authenticate_admin from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.instance_state import get_instance_state from dump_things_service.exceptions import ConfigError +from dump_things_service.instance_state import get_instance_state from dump_things_service.manifest import manifest_configuration from dump_things_service.utils import wrap_http_exception - logger = logging.getLogger('dump_things_service') router = APIRouter() @@ -71,11 +71,10 @@ def get_token_parts(token: str) -> list[str]: status_code=HTTP_201_CREATED, ) async def create_token( - response: Response, - body: TokenRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: TokenRequest, + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> TokenRequest: - token_request = create_or_replace_token(body, api_key, allow_replace=False) response.headers['Location'] = f'/tokens/{quote(body.name)}' return token_request @@ -88,23 +87,21 @@ async def create_token( status_code=HTTP_201_CREATED, ) async def replace_token( - response: Response, - body: TokenRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: TokenRequest, + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> TokenRequest: - token_request = create_or_replace_token(body, api_key, allow_replace=True) response.headers['Location'] = f'/tokens/{quote(body.name)}' return token_request def create_or_replace_token( - body: TokenRequest, - api_key: str, - *, - allow_replace: bool, + body: TokenRequest, + api_key: str, + *, + allow_replace: bool, ) -> TokenRequest: - instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -118,7 +115,7 @@ def create_or_replace_token( ) # Ensure that all specified collections and modes exist - for collection_name, token_collection_info in body.collections.items(): + for collection_name, token_collection_info in body.collections.items(): if collection_name not in abstract_config.collections: detail = f"No such collection: '{collection_name}'." raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=detail) @@ -126,12 +123,11 @@ def create_or_replace_token( # Check that incoming areas are defined if the token allows writing. token_permissions = get_token_permissions(token_collection_info.mode) if token_permissions.incoming_write or token_permissions.zones_access: - # Check for incoming definition in collection config collection_info = abstract_config.collections[collection_name] if not collection_info.incoming: detail = ( - f"Cannot add token with write access to collection " + f'Cannot add token with write access to collection ' f"'{collection_name}' without `incoming`." ) raise HTTPException( @@ -154,7 +150,7 @@ def create_or_replace_token( token_representation=body.representation, ) if existing_token_info: - detail= f"Token with identical representation already exists." + detail = 'Token with identical representation already exists.' raise HTTPException(status_code=HTTP_409_CONFLICT, detail=detail) else: # Generate a random representation that does not yet exist. @@ -203,9 +199,8 @@ def create_or_replace_token( name='Get existing tokens', ) async def get_tokens( - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> list[TokenRequest]: - instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -229,10 +224,9 @@ async def get_tokens( name='Get token by name', ) async def get_token_with_name( - token_name: str, - api_key: str = Depends(api_key_header_scheme), + token_name: str, + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> TokenRequest: - instance_state = get_instance_state() abstract_config = get_config() @@ -259,10 +253,9 @@ async def get_token_with_name( name='Delete token with name', ) async def delete_token_with_name( - token_name: str, - api_key: str = Depends(api_key_header_scheme), + token_name: str, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): - instance_state = get_instance_state() abstract_config = get_config() @@ -294,8 +287,8 @@ async def delete_token_with_name( status_code=HTTP_201_CREATED, ) async def create_admin_token( - body: AdminTokenRequest, - api_key: str = Depends(api_key_header_scheme), + body: AdminTokenRequest, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return create_or_replace_admin_token(body, api_key, allow_replace=False) @@ -307,17 +300,17 @@ async def create_admin_token( status_code=HTTP_201_CREATED, ) async def replace_admin_token( - body: AdminTokenRequest, - api_key: str = Depends(api_key_header_scheme), + body: AdminTokenRequest, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return create_or_replace_admin_token(body, api_key, allow_replace=True) def create_or_replace_admin_token( - body: AdminTokenRequest, - api_key: str, - *, - allow_replace: bool, + body: AdminTokenRequest, + api_key: str, + *, + allow_replace: bool, ): # Check for conflicting token-name if body.name == '__bootstrap__': @@ -328,11 +321,11 @@ def create_or_replace_admin_token( # Check for token content if not body.representation: - detail='Empty administrator token is not allowed' + detail = 'Empty administrator token is not allowed' raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) if not hash_matcher.match(body.representation.strip()): - detail='Hashed token is not a 64-digits hex-number' + detail = 'Hashed token is not a 64-digits hex-number' raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) instance_state = get_instance_state() @@ -369,7 +362,7 @@ def create_or_replace_admin_token( name='Get admin token names', ) async def get_admin_token( - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> list[dict]: instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -377,10 +370,7 @@ async def get_admin_token( authenticate_admin(instance_state, abstract_config, api_key) return [ - { - 'name': token_name, - **(token_value.model_dump(mode='json', by_alias=True)) - } + {'name': token_name, **(token_value.model_dump(mode='json', by_alias=True))} for token_name, token_value in abstract_config.admin_tokens.items() ] + ( [] @@ -400,10 +390,9 @@ async def get_admin_token( name='Delete admin token with name', ) async def delete_admin_token( - token_name: str, - api_key: str = Depends(api_key_header_scheme), + token_name: str, + api_key: Annotated[str, Depends(api_key_header_scheme)], ): - instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index ebb7d69..97d4e5e 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -6,6 +6,7 @@ To speed up processing, multiple indices could be introduced, e.g.: - token representation -> token name """ + from __future__ import annotations import logging @@ -31,12 +32,10 @@ from dump_things_service.abstract_config import ( Configuration, TokenModes, TokenPermission, - mode_mapping, check_collection, get_collection_config_by_name, - get_default_token_config, get_mapping_function_by_name, - get_token_config_for_representation_and_collection, + mode_mapping, ) from dump_things_service.auth import ( AuthenticationError, @@ -83,7 +82,7 @@ def cleaned_json(data: JSON, remove_keys: tuple[str, ...] = ('@type',)) -> JSON: return { key: cleaned_json(value, remove_keys) for key, value in data.items() - if key not in remove_keys and data[key] is not None + if key not in remove_keys and value is not None } return data @@ -97,7 +96,7 @@ def combine_ttl(documents: list[str]) -> str: def wrap_http_exception( exception_class: type[BaseException] = ValueError, status_code: int = HTTP_400_BAD_REQUEST, - header: str = '' + header: str = '', ): """Wrap exceptions of class `exception_class` into HTTP exceptions""" try: @@ -110,12 +109,11 @@ def wrap_http_exception( def join_default_token_permissions( - abstract_configuration: Configuration, - instance_state: InstanceState, - permissions: TokenPermission, - collection: str, + abstract_configuration: Configuration, + instance_state: InstanceState, + permissions: TokenPermission, + collection: str, ) -> TokenPermission: - result = permissions.model_copy() # Get the default token name. If a default token is not defined, return @@ -134,46 +132,41 @@ def join_default_token_permissions( if collection not in abstract_configuration.tokens[default_token_name].collections: return result - default_token_mode = abstract_configuration.tokens[default_token_name].collections[collection].mode + default_token_mode = ( + abstract_configuration.tokens[default_token_name].collections[collection].mode + ) default_token_permissions = mode_mapping[TokenModes(default_token_mode)] result.curated_read = ( - permissions.curated_read | default_token_permissions.curated_read + permissions.curated_read | default_token_permissions.curated_read ) result.incoming_read = ( - permissions.incoming_read | default_token_permissions.incoming_read + permissions.incoming_read | default_token_permissions.incoming_read ) result.incoming_write = ( - permissions.incoming_write | default_token_permissions.incoming_write + permissions.incoming_write | default_token_permissions.incoming_write ) return result def get_on_disk_labels( - store_path: Path, - abstract_config: Configuration, - collection: str, + store_path: Path, + abstract_config: Configuration, + collection: str, ) -> set[str]: check_collection(abstract_config, collection) - incoming_path = ( - store_path / abstract_config.collections[collection].incoming - ) + incoming_path = store_path / abstract_config.collections[collection].incoming if not incoming_path or not incoming_path.exists(): return set() - return { - path.name - for path in incoming_path.iterdir() - if path.is_dir() - } + return {path.name for path in incoming_path.iterdir() if path.is_dir()} def authenticate_token( - instance_state: InstanceState, - collection_name: str, - token_representation: str, + instance_state: InstanceState, + collection_name: str, + token_representation: str, ) -> AuthenticationInfo: - # Try to authenticate the token with the authentication providers that # are associated with the collection. auth_info = None @@ -206,9 +199,9 @@ def authenticate_token( def get_default_token_auth_info( - abstract_config: Configuration, - collection_name: str, - token_name: str, + abstract_config: Configuration, + collection_name: str, + token_name: str, ) -> AuthenticationInfo: token_config = abstract_config.tokens[token_name] collection_info = token_config.collections.get(collection_name) @@ -220,20 +213,19 @@ def get_default_token_auth_info( ) return AuthenticationInfo( token_permission=mode_mapping[TokenModes(collection_info.mode)], - user_id = token_config.user_id, - incoming_label = collection_info.incoming_label, + user_id=token_config.user_id, + incoming_label=collection_info.incoming_label, ) def get_token_store( - abstract_config: Configuration, - instance_state: InstanceState, - collection_name: str, - token_representation: str | None, - *, - is_token_name: bool = False, + abstract_config: Configuration, + instance_state: InstanceState, + collection_name: str, + token_representation: str | None, + *, + is_token_name: bool = False, ) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]: - # If a token representation is provided, try to authenticate the token # with the authentication providers that are associated with the collection. if not is_token_name: @@ -279,11 +271,13 @@ def get_token_store( if not incoming: raise HTTPException( status_code=HTTP_401_UNAUTHORIZED, - detail='No incoming area for collection ' + collection_name + detail='No incoming area for collection ' + collection_name, ) # Check whether a store for this collection and token does already exist. - store_info = instance_state.incoming_stores[collection_name].get(token_representation) + store_info = instance_state.incoming_stores[collection_name].get( + token_representation + ) if store_info: return store_info @@ -304,11 +298,13 @@ def get_token_store( def create_store( - abstract_configuration: Configuration, - instance_state: InstanceState, - collection_name: str, + abstract_configuration: Configuration, + instance_state: InstanceState, + collection_name: str, ) -> _ModelStore: - collection_curated_path = abstract_configuration.collections[collection_name].curated + collection_curated_path = abstract_configuration.collections[ + collection_name + ].curated return create_token_store( abstract_configuration=abstract_configuration, instance_state=instance_state, @@ -318,13 +314,13 @@ def create_store( def create_token_store( - abstract_configuration: Configuration, - instance_state: InstanceState, - collection_name: str, - store_dir: Path, + abstract_configuration: Configuration, + instance_state: InstanceState, + collection_name: str, + store_dir: Path, ) -> _ModelStore: - from dump_things_service.backends.schema_type_layer import SchemaTypeLayer from dump_things_service.abstract_config import get_backend_and_extension + from dump_things_service.backends.schema_type_layer import SchemaTypeLayer from dump_things_service.exceptions import ConfigError from dump_things_service.store.model_store import ModelStore @@ -354,7 +350,6 @@ def create_token_store( backend_config = abstract_configuration.collections[collection_name].backend backend_name, extension = get_backend_and_extension(backend_config.type) if backend_name == 'record_dir': - backend = create_record_dir_token_store_backend( store_dir=store_dir, order_by=instance_state.order_by, @@ -376,7 +371,9 @@ def create_token_store( if extension == 'stl': backend = SchemaTypeLayer(backend=backend, schema=schema_uri) - submission_tags = abstract_configuration.collections[collection_name].submission_tags + submission_tags = abstract_configuration.collections[ + collection_name + ].submission_tags return ModelStore( schema=schema_uri, backend=backend, @@ -388,14 +385,14 @@ def create_token_store( def create_record_dir_token_store_backend( - store_dir: Path, - order_by: list[str], - schema_uri: str, - mapping_function: str, - suffix: str, + store_dir: Path, + order_by: list[str], + schema_uri: str, + mapping_function: str, + suffix: str, ) -> _RecordDirStore: - from dump_things_service.instance_state import record_dir_config_file_name from dump_things_service.backends.record_dir import RecordDirStore + from dump_things_service.instance_state import record_dir_config_file_name # Write the configuration to the store, if it does not yet exist. if not (store_dir / record_dir_config_file_name).exists(): @@ -416,15 +413,16 @@ def create_record_dir_token_store_backend( def write_record_dir_config( - path: Path, - mapping_function: str, - schema: str, + path: Path, + mapping_function: str, + schema: str, ): from dump_things_service.instance_state import record_dir_config_file_name record_dir_config_file_path = path / record_dir_config_file_name if not record_dir_config_file_path.exists(): - record_dir_config_file_path.write_text(f"""# RecordDir Config + record_dir_config_file_path.write_text( + f"""# RecordDir Config type: records version: 1 schema: {schema} @@ -435,9 +433,9 @@ idfx: {mapping_function} def create_sqlite_token_store_backend( - store_dir: Path, - order_by: list[str], -) -> _SQLiteBackend: + store_dir: Path, + order_by: list[str], +) -> _SQLiteBackend: from dump_things_service.backends.sqlite import SQLiteBackend from dump_things_service.backends.sqlite import ( record_file_name as sqlite_record_file_name, @@ -450,26 +448,22 @@ def create_sqlite_token_store_backend( def check_bounds( - length: int | None, - max_length: int, - collection: str, - alternative_url: str + length: int | None, max_length: int, collection: str, alternative_url: str ): if length > max_length: raise HTTPException( status_code=HTTP_413_CONTENT_TOO_LARGE, detail=f"Too many records found in collection '{collection}'. " - f'Please use pagination (/{collection}{alternative_url}).', + f'Please use pagination (/{collection}{alternative_url}).', ) async def process_token( - abstract_config: Configuration, - instance_state: InstanceState, - api_key: str | None, - collection: str, + abstract_config: Configuration, + instance_state: InstanceState, + api_key: str | None, + collection: str, ) -> tuple[TokenPermission, _ModelStore]: - if api_key is None: collection_config = get_collection_config_by_name(abstract_config, collection) token_store, token_permissions, user_id = get_token_store( @@ -480,7 +474,7 @@ async def process_token( is_token_name=True, ) else: - token_store, token_permissions, user_id = get_token_store( + token_store, token_permissions, _user_id = get_token_store( abstract_config, instance_state, collection, @@ -492,16 +486,15 @@ async def process_token( ) # Check for maintenance mode - if collection in instance_state.maintenance_mode: - if not ( - final_permissions.curated_read - and final_permissions.curated_write - and final_permissions.zones_access - ): - raise HTTPException( - status_code=HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Collection '{collection}' is in maintenance mode", - ) + if collection in instance_state.maintenance_mode and not ( + final_permissions.curated_read + and final_permissions.curated_write + and final_permissions.zones_access + ): + raise HTTPException( + status_code=HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Collection '{collection}' is in maintenance mode", + ) if not final_permissions.incoming_read and not final_permissions.curated_read: raise HTTPException( @@ -512,32 +505,26 @@ async def process_token( def get_required_incoming_labels( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> set[str]: - return set( - map( - lambda x: x[1], - get_required_incoming_info(abstract_config, collection_name), - ) - ) + return {x[1] for x in get_required_incoming_info(abstract_config, collection_name)} def get_required_incoming_info( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> set[tuple[str, str]]: return { (token_name, this_collection_info.incoming_label) for token_name, token_info in abstract_config.tokens.items() for this_collection_name, this_collection_info in token_info.collections.items() - if this_collection_name == collection_name and mode_mapping[ - TokenModes(this_collection_info.mode) - ].incoming_write is True + if this_collection_name == collection_name + and mode_mapping[TokenModes(this_collection_info.mode)].incoming_write is True } def var_escape( - name: str, + name: str, ) -> str: return name.replace('_', '___').replace('-', '_0_') diff --git a/dump_things_service/validate.py b/dump_things_service/validate.py index eae50f2..4f80e31 100644 --- a/dump_things_service/validate.py +++ b/dump_things_service/validate.py @@ -33,15 +33,14 @@ from dump_things_service.utils import ( def validate_record( - collection: str, - data: BaseModel | str, - class_name: str, - model: Any, - input_format: Format, - _: bool, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + data: BaseModel | str, + class_name: str, + model: Any, + input_format: Format, + _: bool, + api_key: str | None = Depends(api_key_header_scheme), ) -> JSONResponse: - instance_state = get_instance_state() abstract_config = get_config() @@ -63,7 +62,7 @@ def validate_record( else api_key ) - store, token_permissions, user_id = get_token_store( + _store, token_permissions, _user_id = get_token_store( abstract_config, instance_state, collection, @@ -82,18 +81,30 @@ def validate_record( ) if input_format == Format.ttl: - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Conversion error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Conversion error', + ): json_object = FormatConverter( abstract_config.collections[collection].schema_location, input_format=Format.ttl, output_format=Format.json, ).convert(data, class_name) - with wrap_http_exception(ValidationError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValidationError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): TypeAdapter(getattr(model, class_name)).validate_python(json_object) else: # Try to convert it into TTL to detect potential errors before storing # the record - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(data) return JSONResponse(True) diff --git a/pyproject.toml b/pyproject.toml index 060475e..e1358a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [build-system] -requires = ["hatchling"] +requires = [ + "hatchling", + "hatch-vcs", +] build-backend = "hatchling.build" [project] @@ -17,9 +20,7 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Development Status :: 4 - Beta", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", @@ -42,9 +43,20 @@ dependencies = [ ] [project.urls] -Documentation = "https://hub.psychoinformatics.de/datalink/dump-things-server" +Documentation = "https://hub.psychoinformatics.de/orinoco/dump-things-server" Issues = "https://codeberg.org/datalink/dump-things-server/issues" -Source = "https://hub.psychoinformatics.de/datalink/dump-things-server" +Source = "https://hub.psychoinformatics.de/orinoco/dump-things-server" +Changelog = "https://hub.psychoinformatics.de/orinoco/dump-things-server/src/branch/master/CHANGELOG.md" + +[project.optional-dependencies] +# this is what readthedocs consumes to decide what needs to be installed +# for compiling the docs +docs = [ + "pytest", + "sphinx", + "sphinx_rtd_theme", + "sphinx_autodoc_typehints", +] [project.scripts] dump-things-service = "dump_things_service.main:main" @@ -75,14 +87,36 @@ only-include = [ ] [tool.hatch.version] -path = "dump_things_service/__about__.py" +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "dump_things_service/_version.py" [tool.hatch.envs.types] extra-dependencies = [ "mypy>=1.0.0", ] [tool.hatch.envs.types.scripts] -check = "mypy --install-types --non-interactive {args:src tests}" +check = "mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context {args:dump_things_service}" + +[tool.hatch.envs.docs] +description = "build Sphinx-based docs" +# also see project.optional-dependencies.docs! +# this is not considered by readthedocs +extra-dependencies = [ + "pytest", + "sphinx", + "sphinx_rtd_theme", + "sphinx-autodoc-typehints", +] +[tool.hatch.envs.docs.scripts] +build = [ + "make -C docs html", +] +clean = [ + "rm -rf docs/generated", + "make -C docs clean", +] [tool.coverage.run] source_pkgs = ["dump_things_service"] @@ -106,21 +140,19 @@ description = "fastapi dev environment" [tool.hatch.envs.fastapi.scripts] run = "python -m dump_things_service.main {args}" -[[tool.hatch.envs.tests.matrix]] +[[tool.hatch.envs.hatch-test.matrix]] python = ["3.11", "3.12"] -[tool.hatch.envs.tests] +[tool.hatch.envs.hatch-test] +default-args = ["dump_things_service"] extra-dependencies = [ "freezegun", - "httpx", + "httpx2", "pytest", "pytest-cov", "pytest-httpserver", ] -[tool.hatch.envs.tests.scripts] -run = 'python -m pytest {args}' - [tool.ruff] extend-exclude = [ # sphinx @@ -130,7 +162,7 @@ extend-exclude = [ ] line-length = 88 indent-width = 4 -target-version = "py39" +target-version = "py311" [tool.ruff.format] # Prefer single quotes over double quotes. quote-style = "single" @@ -152,3 +184,6 @@ skip = '.git*' check-hidden = true # ignore-regex = '' # ignore-words-list = '' + +[tool.mypy] +disable_error_code = ["import-untyped"]