From ea2faed30fd00fc717bf8f0902fd277bed5bec9b Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 17:29:05 +0200 Subject: [PATCH 01/19] chore: vim user friendly .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fba313f..26e292c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/** tmp/** **/__pycache__ **/.hypothesis +.*.swp -- 2.52.0 From 77063de244c44af134dd25ef5d47f918f036e84a Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 17:32:41 +0200 Subject: [PATCH 02/19] chore: uniform use of Python 3.11 as minimum version --- pyproject.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 060475e..1034139 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,7 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Development Status :: 4 - Beta", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", @@ -82,7 +80,7 @@ extra-dependencies = [ "mypy>=1.0.0", ] [tool.hatch.envs.types.scripts] -check = "mypy --install-types --non-interactive {args:src tests}" +check = "mypy --install-types --non-interactive --python-version 3.11 {args:src tests}" [tool.coverage.run] source_pkgs = ["dump_things_service"] @@ -130,7 +128,7 @@ extend-exclude = [ ] line-length = 88 indent-width = 4 -target-version = "py39" +target-version = "py311" [tool.ruff.format] # Prefer single quotes over double quotes. quote-style = "single" -- 2.52.0 From 7d421c8cf6532c920144f9c52515dc9b082027fd Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 17:43:19 +0200 Subject: [PATCH 03/19] chore: functional mypy setup, now shows 109 errors --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1034139..35f9bdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ extra-dependencies = [ "mypy>=1.0.0", ] [tool.hatch.envs.types.scripts] -check = "mypy --install-types --non-interactive --python-version 3.11 {args:src tests}" +check = "mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context {args:dump_things_service}" [tool.coverage.run] source_pkgs = ["dump_things_service"] -- 2.52.0 From 736e12012e2fba534981cb75d908a5a023f87038 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 17:48:28 +0200 Subject: [PATCH 04/19] chore: migrate from `hatch run tests:run` to `hatch test` This is arguably more standard and what a random `hatch` user would expect. I saw no specific reasons to deviate from this standard. --- .forgejo/workflows/run_tests.yaml | 6 +++--- pyproject.toml | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.forgejo/workflows/run_tests.yaml b/.forgejo/workflows/run_tests.yaml index 6630ca2..f13411c 100644 --- a/.forgejo/workflows/run_tests.yaml +++ b/.forgejo/workflows/run_tests.yaml @@ -23,14 +23,14 @@ jobs: - name: Run tests run: | - hatch run tests:run \ + hatch test \ --ignore=dump_things_service/tests/test_generators.py \ --ignore=dump_things_service/tests/test_ifabsent_patch.py - name: Run generator tests run: | - hatch run tests:run dump_things_service/tests/test_generators.py + hatch test dump_things_service/tests/test_generators.py - name: Run ifabsent-patch tests run: | - hatch run tests:run dump_things_service/tests/test_ifabsent_patch.py + hatch test dump_things_service/tests/test_ifabsent_patch.py diff --git a/pyproject.toml b/pyproject.toml index 35f9bdc..3daff3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,10 +104,11 @@ description = "fastapi dev environment" [tool.hatch.envs.fastapi.scripts] run = "python -m dump_things_service.main {args}" -[[tool.hatch.envs.tests.matrix]] +[[tool.hatch.envs.hatch-test.matrix]] python = ["3.11", "3.12"] -[tool.hatch.envs.tests] +[tool.hatch.envs.hatch-test] +default-args = ["dump_things_service"] extra-dependencies = [ "freezegun", "httpx", @@ -116,9 +117,6 @@ extra-dependencies = [ "pytest-httpserver", ] -[tool.hatch.envs.tests.scripts] -run = 'python -m pytest {args}' - [tool.ruff] extend-exclude = [ # sphinx -- 2.52.0 From 45aeaf6607757e7bceb6face058a254350249626 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 17:53:52 +0200 Subject: [PATCH 05/19] ci: remove dedicated Python installation There should be no need for that. `uv` can manage this on its own. This has the advantage that the outcome, when run locally, is likely to be more similar than what that action might do. This may remove unexplained behavior differences between CI results and local tests. --- .forgejo/workflows/run_tests.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.forgejo/workflows/run_tests.yaml b/.forgejo/workflows/run_tests.yaml index f13411c..03a2241 100644 --- a/.forgejo/workflows/run_tests.yaml +++ b/.forgejo/workflows/run_tests.yaml @@ -12,9 +12,6 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - - name: Install uv uses: astral-sh/setup-uv@v6 -- 2.52.0 From d25e58d0a54bbb060b710bfc82ac96cbee292ea8 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 17:59:59 +0200 Subject: [PATCH 06/19] chore: migrate from `httpx` -> `httpx2` for tests Reason: ``` StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. from starlette.testclient import TestClient ``` Closes: https://hub.psychoinformatics.de/orinoco/dump-things-server/issues/243 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3daff3f..330cef4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,7 +111,7 @@ python = ["3.11", "3.12"] default-args = ["dump_things_service"] extra-dependencies = [ "freezegun", - "httpx", + "httpx2", "pytest", "pytest-cov", "pytest-httpserver", -- 2.52.0 From 23baed424e8168e5e511ecc43301f57af32b4bfd Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 18:03:00 +0200 Subject: [PATCH 07/19] chore: update project URLs, include changelog URL --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 330cef4..95b152e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,10 @@ dependencies = [ ] [project.urls] -Documentation = "https://hub.psychoinformatics.de/datalink/dump-things-server" +Documentation = "https://hub.psychoinformatics.de/orinoco/dump-things-server" Issues = "https://codeberg.org/datalink/dump-things-server/issues" -Source = "https://hub.psychoinformatics.de/datalink/dump-things-server" +Source = "https://hub.psychoinformatics.de/orinoco/dump-things-server" +Changelog = "https://hub.psychoinformatics.de/orinoco/dump-things-server/src/branch/master/CHANGELOG.md" [project.scripts] dump-things-service = "dump_things_service.main:main" -- 2.52.0 From a07f9117afba6f14479f7238446fdf211df589f6 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 19:43:12 +0200 Subject: [PATCH 08/19] chore: switch to VCS-based version identifier generation --- .gitignore | 1 + dump_things_service/__init__.py | 3 +++ pyproject.toml | 10 ++++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 26e292c..6351760 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ tmp/** **/__pycache__ **/.hypothesis .*.swp +dump_things_service/_version.py diff --git a/dump_things_service/__init__.py b/dump_things_service/__init__.py index cdb6a1a..88d9f3d 100644 --- a/dump_things_service/__init__.py +++ b/dump_things_service/__init__.py @@ -20,7 +20,10 @@ from starlette.status import ( HTTP_503_SERVICE_UNAVAILABLE, ) +from dump_things_service._version import __version__ + __all__ = [ + '__version__', 'Format', 'HTTP_200_OK', 'HTTP_201_CREATED', diff --git a/pyproject.toml b/pyproject.toml index 95b152e..1d6afbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [build-system] -requires = ["hatchling"] +requires = [ + "hatchling", + "hatch-vcs", +] build-backend = "hatchling.build" [project] @@ -74,7 +77,10 @@ only-include = [ ] [tool.hatch.version] -path = "dump_things_service/__about__.py" +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "dump_things_service/_version.py" [tool.hatch.envs.types] extra-dependencies = [ -- 2.52.0 From 018f5c45d5cb57b902b985d160fc3455f2d2c2c4 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 19:45:43 +0200 Subject: [PATCH 09/19] chore: add framework for Sphinx-based docs This is working towards a more structured documentation. See: https://hub.psychoinformatics.de/orinoco/dump-things-server/issues/214 --- docs/.gitignore | 2 ++ docs/Makefile | 20 ++++++++++++++++++ docs/_static/.gitkeep | 0 docs/conf.py | 48 +++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 10 +++++++++ pyproject.toml | 29 ++++++++++++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 docs/.gitignore create mode 100644 docs/Makefile create mode 100644 docs/_static/.gitkeep create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..9719ae4 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +_build +generated diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..973b097 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= --fail-on-warning +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..b79be80 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,48 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +import dump_things_service + +project = 'dump-things-server' +copyright = '2025-2026, Christian Mönch' +author = 'Christian Mönch' +release = dump_things_service.__version__ + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'sphinx.ext.autosummary', + 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', + 'sphinx.ext.viewcode', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +primary_domain = 'py' +autoclass_content = "both" + +typehints_use_signature = True +typehints_use_signature_return = True + +# we build some docstrings from loguru. define some no-op substitutions +# to avoid errors +rst_prolog = """ +.. |Logger| replace:: Logger +.. |add| replace:: add +.. |sys.stderr| replace:: sys.stderr +.. |str.format| replace:: str.format +""" + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..17022f1 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,10 @@ +The `dump-thing-server` documentation +===================================== + +HERE BE CONTENT... + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/pyproject.toml b/pyproject.toml index 1d6afbd..f365703 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,16 @@ Issues = "https://codeberg.org/datalink/dump-things-server/issues" Source = "https://hub.psychoinformatics.de/orinoco/dump-things-server" Changelog = "https://hub.psychoinformatics.de/orinoco/dump-things-server/src/branch/master/CHANGELOG.md" +[project.optional-dependencies] +# this is what readthedocs consumes to decide what needs to be installed +# for compiling the docs +docs = [ + "pytest", + "sphinx", + "sphinx_rtd_theme", + "sphinx_autodoc_typehints", +] + [project.scripts] dump-things-service = "dump_things_service.main:main" dump-things-rebuild-index = "dump_things_service.commands.rebuild_index:main" @@ -89,6 +99,25 @@ extra-dependencies = [ [tool.hatch.envs.types.scripts] check = "mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context {args:dump_things_service}" +[tool.hatch.envs.docs] +description = "build Sphinx-based docs" +# also see project.optional-dependencies.docs! +# this is not considered by readthedocs +extra-dependencies = [ + "pytest", + "sphinx", + "sphinx_rtd_theme", + "sphinx-autodoc-typehints", +] +[tool.hatch.envs.docs.scripts] +build = [ + "make -C docs html", +] +clean = [ + "rm -rf docs/generated", + "make -C docs clean", +] + [tool.coverage.run] source_pkgs = ["dump_things_service"] branch = true -- 2.52.0 From 9c6c366b407eb5bf972fade57efce326fea8e879 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 19:53:10 +0200 Subject: [PATCH 10/19] ci: make codespell workflow run on forgejo --- .forgejo/workflows/codespell.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/codespell.yml b/.forgejo/workflows/codespell.yml index 93e4437..5b2ce9f 100644 --- a/.forgejo/workflows/codespell.yml +++ b/.forgejo/workflows/codespell.yml @@ -2,7 +2,7 @@ --- name: Codespell -on: workflow_dispatch +on: [push, pull_request, workflow_dispatch] permissions: contents: read @@ -10,13 +10,13 @@ permissions: jobs: codespell: name: Check for spelling errors - runs-on: ubuntu-latest + runs-on: debian-latest steps: - name: Checkout uses: actions/checkout@v5 - name: Codespell - uses: codespell-project/actions-codespell@v2 + uses: https://github.com/codespell-project/actions-codespell@v2 with: ignore_words_list: crate -- 2.52.0 From a9f3ad5537d1ee5767cfe861849fb21026875ed5 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 19:57:22 +0200 Subject: [PATCH 11/19] chore: fix typo pointed out by codespell --- CHANGELOG.md | 2 +- README.md | 2 +- dump_things_service/model.py | 2 +- dump_things_service/tests/test_token_endpoints.py | 4 ++-- dump_things_service/tests/test_unicode.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60a3533..e78b17d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,7 +123,7 @@ 3. The top-level mapping `admin_tokens` was added. - Configuration files are no longer read when the service is started. Instead - the service reads its configuration from the store, if it is present. Thw tool + the service reads its configuration from the store, if it is present. The tool (`dump-things-load-config`) can read an existing configuration file and manifest the described configuration on a running dump-things server. It supports pre version 6 config files and converts them to the new diff --git a/README.md b/README.md index ec6cc93..e208c9c 100644 --- a/README.md +++ b/README.md @@ -594,7 +594,7 @@ Most endpoints require a *collection*. These correspond to the names of the "dat The service provides the following user endpoints (In addition to user endpoints, there exist endpoints for curators. To view them, check the `/docs`-path in an installed service): -- `POST /maintenance`: this endpoint allows to set a collection into mantenance mode. +- `POST /maintenance`: this endpoint allows to set a collection into maintenance mode. In maintenance mode, only tokens with curator-privileges can access the collection. The posted data is a JSON that contains the name of the collection and whether the maintenance state should be active or not, for example: ```json diff --git a/dump_things_service/model.py b/dump_things_service/model.py index 9155490..4ea5750 100644 --- a/dump_things_service/model.py +++ b/dump_things_service/model.py @@ -67,7 +67,7 @@ def get_subclasses( # TODO: shall we use the following code? # The code below would use schema-definitions to determine classes and not -# go through thw pydantic module generation. +# go through the pydantic module generation. @cache def get_subclasses_2( collection_name: str, diff --git a/dump_things_service/tests/test_token_endpoints.py b/dump_things_service/tests/test_token_endpoints.py index 0209eb3..20ece81 100644 --- a/dump_things_service/tests/test_token_endpoints.py +++ b/dump_things_service/tests/test_token_endpoints.py @@ -18,7 +18,7 @@ def test_token_creation(fastapi_client_simple): } } - # Create a token eith name 'a' + # Create a token with name 'a' response = test_client.post( '/tokens', headers={'x-dumpthings-token': admin_token}, @@ -34,7 +34,7 @@ def test_token_creation(fastapi_client_simple): ) assert response.status_code == HTTP_409_CONFLICT - # Try to create another token eith name 'b' and the same representation + # Try to create another token with name 'b' and the same representation # as 'a', should result in a 4ß9-error json_record['name'] = 'b' response = test_client.post( diff --git a/dump_things_service/tests/test_unicode.py b/dump_things_service/tests/test_unicode.py index 15cf7cc..68ce9fe 100644 --- a/dump_things_service/tests/test_unicode.py +++ b/dump_things_service/tests/test_unicode.py @@ -33,7 +33,7 @@ def test_unicode_iri(fastapi_client_simple): headers={'x-dumpthings-token': 'token-1'}, json = { 'pid': 'https://en.wikipedia.org/wiki/Universita_degli_Studi_eCampus', - 'given_name': 'Università degli Studi eCampus (Italy)', + 'given_name': 'Università degli Studi eCampus (Italy)', # codespell:ignore } ) assert response.status_code == HTTP_200_OK -- 2.52.0 From 0534a14e949438f2f459a741cca56f389bfc4369 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 20:04:26 +0200 Subject: [PATCH 12/19] ci: add code linting workflow There is a substantial number of issues reported, even with the config that was already present in `pyproject.toml`. This changeset only adds the workflow. Fixing there issues needs to be done later, possibly in an incremental fashion. --- .forgejo/workflows/ruff.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .forgejo/workflows/ruff.yml diff --git a/.forgejo/workflows/ruff.yml b/.forgejo/workflows/ruff.yml new file mode 100644 index 0000000..1b0ddc1 --- /dev/null +++ b/.forgejo/workflows/ruff.yml @@ -0,0 +1,17 @@ +name: Ruff +on: [push, pull_request] +jobs: + ruff: + name: Code linting + runs-on: debian-latest + steps: + - name: Checkout project + uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install hatch + run: uv tool install hatch + - name: Check code + run: hatch check code + - name: Check formatting + run: hatch check fmt -- 2.52.0 From b9157d5de24b0e72bb69e403066a77395aa04ee3 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 20:08:17 +0200 Subject: [PATCH 13/19] ci: add workflow to check typing in files changed in a given PR Right now, it makes no sense to type-check the entire project due to the large number of issues. This is an alternative approach that should help improve the situation over time. --- .forgejo/workflows/mypy-pr.yml | 36 ++++++++++++++++++++++++++++++++++ pyproject.toml | 3 +++ 2 files changed, 39 insertions(+) create mode 100644 .forgejo/workflows/mypy-pr.yml diff --git a/.forgejo/workflows/mypy-pr.yml b/.forgejo/workflows/mypy-pr.yml new file mode 100644 index 0000000..024c185 --- /dev/null +++ b/.forgejo/workflows/mypy-pr.yml @@ -0,0 +1,36 @@ +name: Type annotation (PR) + +on: + pull_request: + paths: + - 'dump_things_service/**.py' + - '!**/tests/**.py' + +jobs: + check: + runs-on: debian-latest + steps: + - name: Checkout project + uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install hatch + run: uv tool install hatch + - name: Get Python changed files + id: changed-py-files + uses: https://github.com/tj-actions/changed-files@v46 + with: + files: | + *.py + **/*.py + - name: Type check changed files + if: steps.changed-py-files.outputs.any_changed == 'true' + run: | + # get any type stubs that mypy thinks it needs + # run mypy on the modified files only, and do not even follow imports. + # this results is a fairly superficial test, but given the overall + # state of annotations, we strive to become more correct incrementally + # with focused error reports, rather than barfing a huge complaint + # that is unrelated to the changeset someone has been working on. + # run on the oldest supported Python version + hatch run types:mypy --install-types --non-interactive --python-version 3.11 --follow-imports skip --pretty --show-error-context ${{ steps.changed-py-files.outputs.all_changed_files }} diff --git a/pyproject.toml b/pyproject.toml index f365703..e1358a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,3 +184,6 @@ skip = '.git*' check-hidden = true # ignore-regex = '' # ignore-words-list = '' + +[tool.mypy] +disable_error_code = ["import-untyped"] -- 2.52.0 From e7599a86b1da9604ba7af3b7cd2d8b9b19a81f53 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 20:33:20 +0200 Subject: [PATCH 14/19] chore: fix `noqa` directive syntax and incomplete issue identifiers --- dump_things_service/collection.py | 14 +++++++------- .../commands/create_merged_schema.py | 2 +- dump_things_service/main.py | 18 +++++++++--------- dump_things_service/model.py | 10 +++++----- dump_things_service/tests/test_roundtrip.py | 2 +- .../tests/test_roundtrip_flatsocial.py | 2 +- .../tests/test_web_interface.py | 6 +++--- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 71c1533..7907459 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -70,12 +70,12 @@ from dump_things_service.utils import ( # This following lines are required for dynamic endpoint generation -from typing import Annotated # noqa 401 -- used by autogenerated code -from fastapi import Body # noqa 401 -- used by autogenerated code -from dump_things_service.api_key import api_key_header_scheme # noqa 401 -- used by autogenerated code -from dump_things_service.curated import store_curated_record # noqa 401 -- used by autogenerated code -from dump_things_service.incoming import store_incoming_record # noqa 401 -- used by autogenerated code -from dump_things_service.validate import validate_record # noqa 401 -- used by autogenerated code +from typing import Annotated # noqa: F401 -- used by autogenerated code +from fastapi import Body # noqa: F401 -- used by autogenerated code +from dump_things_service.api_key import api_key_header_scheme # noqa: F401 -- used by autogenerated code +from dump_things_service.curated import store_curated_record # noqa: F401 -- used by autogenerated code +from dump_things_service.incoming import store_incoming_record # noqa: F401 -- used by autogenerated code +from dump_things_service.validate import validate_record # noqa: F401 -- used by autogenerated code logger = logging.getLogger('dump_things_service') @@ -435,7 +435,7 @@ def create_endpoint( info=f"'{operation_name} {collection_name}/{class_name} objects'", handler=handler, ) - exec(endpoint_source, globals()) # noqa S102 + exec(endpoint_source, globals()) # noqa: S102 # Create an API route for the endpoint app.add_api_route( diff --git a/dump_things_service/commands/create_merged_schema.py b/dump_things_service/commands/create_merged_schema.py index 859a413..d094765 100644 --- a/dump_things_service/commands/create_merged_schema.py +++ b/dump_things_service/commands/create_merged_schema.py @@ -5,7 +5,7 @@ import yaml from linkml_runtime.utils.schemaview import SchemaView # Patch linkml -from dump_things_service.patches import enabled # noqa F401 -- patches LinkML +from dump_things_service.patches import enabled # noqa: F401 -- patches LinkML parser = ArgumentParser( prog='Create a static schema with all imported schemas integrated', diff --git a/dump_things_service/main.py b/dump_things_service/main.py index 7803ffc..68cc747 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -11,7 +11,7 @@ from dump_things_service.abstract_config import store_config from dump_things_service.commands.upload_config import convert_config_1_to_config_2 from dump_things_service.manifest import manifest_configuration # Perform the patching before importing any third-party libraries -from dump_things_service.patches import enabled # noqa F401 -- used by generated code +from dump_things_service.patches import enabled # noqa: F401 -- used by generated code import yaml import uvicorn @@ -106,7 +106,7 @@ logger = logging.getLogger('dump_things_service') parser = argparse.ArgumentParser() -parser.add_argument('--host', default='0.0.0.0') # noqa S104 +parser.add_argument('--host', default='0.0.0.0') # noqa: S104 parser.add_argument('--port', default=8000, type=int) parser.add_argument('--origins', action='append', default=[]) parser.add_argument( @@ -405,7 +405,7 @@ async def maintenance( async def read_record_with_pid( collection: str, pid: str, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ): check_collection(g_configuration, collection) @@ -448,7 +448,7 @@ async def read_record_with_pid( async def read_all_records( collection: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ): return await _read_all_records( @@ -471,7 +471,7 @@ async def read_all_records( async def read_all_records_paginated( collection: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ) -> Page[dict | str]: result_list = await _read_all_records( @@ -493,7 +493,7 @@ async def read_records_of_type( collection: str, class_name: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ): return await _read_records_of_type( @@ -518,7 +518,7 @@ async def read_records_of_type_paginated( collection: str, class_name: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), ) -> Page[dict | str]: result_list = await _read_records_of_type( @@ -535,7 +535,7 @@ async def read_records_of_type_paginated( async def _read_all_records( collection: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), bound: int | None = None, ) -> LazyList: @@ -591,7 +591,7 @@ async def _read_records_of_type( collection: str, class_name: str, matching: str | None = None, - format: Format = Format.json, # noqa A002 + format: Format = Format.json, # noqa: A002 api_key: str = Depends(api_key_header_scheme), bound: int | None = None, ) -> LazyList: diff --git a/dump_things_service/model.py b/dump_things_service/model.py index 4ea5750..d6311a5 100644 --- a/dump_things_service/model.py +++ b/dump_things_service/model.py @@ -1,6 +1,6 @@ from __future__ import annotations -import dataclasses # noqa F401 -- used by generated code +import dataclasses # noqa: F401 -- used by generated code import logging import sys from functools import cache @@ -11,9 +11,9 @@ from typing import ( ) from urllib.parse import urlparse -import annotated_types # noqa F401 -- used by generated code -import pydantic # noqa F401 -- used by generated code -import pydantic_core # noqa F401 -- used by generated code +import annotated_types # noqa: F401 -- used by generated code +import pydantic # noqa: F401 -- used by generated code +import pydantic_core # noqa: F401 -- used by generated code from linkml.generators import ( PydanticGenerator, PythonGenerator, @@ -22,7 +22,7 @@ from linkml_runtime import SchemaView from pydantic._internal._model_construction import ModelMetaclass # Ensure linkml is patched -import dump_things_service.patches.enabled # noqa F401 -- apply patches +import dump_things_service.patches.enabled # noqa: F401 -- apply patches if TYPE_CHECKING: from types import ModuleType diff --git a/dump_things_service/tests/test_roundtrip.py b/dump_things_service/tests/test_roundtrip.py index 50bbb7f..0b9608d 100644 --- a/dump_things_service/tests/test_roundtrip.py +++ b/dump_things_service/tests/test_roundtrip.py @@ -1,5 +1,5 @@ import freezegun -import pytest # noqa F401 +import pytest # noqa: F401 from .. import HTTP_200_OK from ..utils import cleaned_json diff --git a/dump_things_service/tests/test_roundtrip_flatsocial.py b/dump_things_service/tests/test_roundtrip_flatsocial.py index bb2481c..ada9aeb 100644 --- a/dump_things_service/tests/test_roundtrip_flatsocial.py +++ b/dump_things_service/tests/test_roundtrip_flatsocial.py @@ -1,4 +1,4 @@ -import pytest # noqa F401 +import pytest # noqa: F401 import freezegun diff --git a/dump_things_service/tests/test_web_interface.py b/dump_things_service/tests/test_web_interface.py index 8096d5a..0d435bb 100644 --- a/dump_things_service/tests/test_web_interface.py +++ b/dump_things_service/tests/test_web_interface.py @@ -14,7 +14,7 @@ pids = ('', '--------', '&&&&&', 'abc', 'abc&', 'abc&format=ttl') @pytest.mark.parametrize( - 'collection_name,class_name,query,format_name', # noqa PT006 + 'collection_name,class_name,query,format_name', # noqa: PT006 tuple(product(*(collection_names, class_names, queries, format_names))), ) def test_web_interface_post_errors( @@ -35,7 +35,7 @@ def test_web_interface_post_errors( @pytest.mark.parametrize( - 'collection_name,class_name,query,format_name', # noqa PT006 + 'collection_name,class_name,query,format_name', # noqa: PT006 tuple(product(*(collection_names, class_names, queries, format_names))), ) def test_web_interface_get_class_errors( @@ -60,7 +60,7 @@ def test_web_interface_get_class_errors( @pytest.mark.parametrize( - 'collection_name,pid,query,format_name', # noqa PT006 + 'collection_name,pid,query,format_name', # noqa: PT006 tuple(product(*(collection_names, pids, queries, format_names))), ) def test_web_interface_get_pid_errors( -- 2.52.0 From 666ed4b69ecde2fb77fa58e0ce7b05993434bcf8 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 20:37:04 +0200 Subject: [PATCH 15/19] chore: auto-apply code linting fixes via `hatch check code --fix` This fixes >100 issues. The tests continue to pass. --- dump_things_service/__init__.py | 4 +- dump_things_service/abstract_config.py | 11 ++-- dump_things_service/admin.py | 1 - dump_things_service/audit/gitaudit.py | 11 ++-- dump_things_service/auth/config.py | 10 ++-- dump_things_service/auth/forgejo.py | 4 +- dump_things_service/authenticate.py | 20 ------- dump_things_service/backends/record_dir.py | 4 +- .../backends/schema_type_layer.py | 5 +- dump_things_service/backends/sqlite.py | 2 +- dump_things_service/collection.py | 57 +++++++++++-------- dump_things_service/collection_endpoints.py | 10 ++-- dump_things_service/commands/copy_store.py | 2 +- .../commands/download_config.py | 1 - .../commands/gitaudit_rebuild_index.py | 1 - .../commands/gitaudit_report.py | 1 - dump_things_service/commands/hash_token.py | 1 - dump_things_service/commands/rebuild_index.py | 3 +- dump_things_service/commands/upload_config.py | 20 +++---- dump_things_service/converter.py | 5 +- dump_things_service/curated.py | 4 +- dump_things_service/incoming.py | 2 +- dump_things_service/instance_state.py | 6 +- dump_things_service/lazy_list.py | 3 +- dump_things_service/main.py | 33 +++++------ dump_things_service/manifest.py | 1 - dump_things_service/store/model_store.py | 11 ++-- dump_things_service/tests/create_store.py | 13 +++-- dump_things_service/tests/fixtures.py | 15 +++-- dump_things_service/tests/test_basic.py | 5 +- .../tests/test_collection_administration.py | 17 +++--- dump_things_service/tests/test_config.py | 7 +-- dump_things_service/tests/test_curated.py | 8 +-- .../tests/test_ifabsent_patch.py | 1 - .../tests/test_roundtrip_flatsocial.py | 3 +- dump_things_service/tests/test_unicode.py | 1 - dump_things_service/token_endpoints.py | 5 +- dump_things_service/utils.py | 10 ++-- 38 files changed, 137 insertions(+), 181 deletions(-) diff --git a/dump_things_service/__init__.py b/dump_things_service/__init__.py index 88d9f3d..fd2090d 100644 --- a/dump_things_service/__init__.py +++ b/dump_things_service/__init__.py @@ -23,8 +23,6 @@ from starlette.status import ( from dump_things_service._version import __version__ __all__ = [ - '__version__', - 'Format', 'HTTP_200_OK', 'HTTP_201_CREATED', 'HTTP_300_MULTIPLE_CHOICES', @@ -40,6 +38,8 @@ __all__ = [ 'HTTP_503_SERVICE_UNAVAILABLE', 'JSON', 'YAML', + 'Format', + '__version__', 'config_file_name', 'reserved_collection_names', ] diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index 384d90c..b600cf0 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -1,14 +1,13 @@ import enum import hashlib import logging +from collections.abc import Callable, Iterable from functools import partial from pathlib import ( Path, PurePosixPath, ) from typing import ( - Callable, - Iterable, Literal, cast, ) @@ -17,7 +16,8 @@ from fastapi import HTTPException from pydantic import ( BaseModel, ConfigDict, - Field, ValidationError, + Field, + ValidationError, ) from yaml.scanner import ScannerError @@ -27,12 +27,11 @@ from dump_things_service import ( ) from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.backends.record_dir import ( - _RecordDirStore, RecordDirStore, + _RecordDirStore, ) from dump_things_service.exceptions import ConfigError - logger = logging.getLogger('dump_things_service') g_abstract_configuration = None @@ -259,7 +258,7 @@ def get_config() -> Configuration: if not g_abstract_configuration: msg = 'Configuration not yet loaded' raise RuntimeError(msg) - return cast(Configuration, g_abstract_configuration) + return cast('Configuration', g_abstract_configuration) def store_config( diff --git a/dump_things_service/admin.py b/dump_things_service/admin.py index 20a19c0..201e4d9 100644 --- a/dump_things_service/admin.py +++ b/dump_things_service/admin.py @@ -9,7 +9,6 @@ from dump_things_service.abstract_config import ( ) from dump_things_service.instance_state import InstanceState - logger = logging.getLogger('dump_things_service') diff --git a/dump_things_service/audit/gitaudit.py b/dump_things_service/audit/gitaudit.py index b3afaed..fec83dc 100644 --- a/dump_things_service/audit/gitaudit.py +++ b/dump_things_service/audit/gitaudit.py @@ -23,13 +23,12 @@ import yaml from datalad_core.git_utils import apply_changeset from datalad_core.repo import Repo from datalad_core.runners import ( - call_git, CommandError, + call_git, ) from . import AuditBackend - index_file_name = 'gitaudit_index.log' @@ -247,7 +246,7 @@ class GitAuditBackend(AuditBackend): ) -> bytes: try: return call_git( - ['cat-file', '-p', f'master:{str(path)}'], + ['cat-file', '-p', f'master:{path!s}'], cwd=self.path, capture_output=True, ) @@ -321,8 +320,8 @@ class GitAuditBackend(AuditBackend): if not self.index_path.exists(): self._rebuild_index() - with open(self.index_path, 'rt') as f: - self.index = set(line.strip() for line in f.readlines()) + with open(self.index_path) as f: + self.index = set(line.strip() for line in f) def _add_to_index( self, @@ -338,7 +337,7 @@ class GitAuditBackend(AuditBackend): cwd=self.path, capture_output=True, ).decode().splitlines() - with open(self.index_path, 'wt') as f: + with open(self.index_path, 'w') as f: for line in tree_entries: if not line.endswith('.yaml'): continue diff --git a/dump_things_service/auth/config.py b/dump_things_service/auth/config.py index 8f8976e..7aeb45c 100644 --- a/dump_things_service/auth/config.py +++ b/dump_things_service/auth/config.py @@ -1,15 +1,15 @@ """Use configuration information to fetch token permissions, ids, and incoming_label """ -from dump_things_service.abstract_config import Configuration +from dump_things_service.abstract_config import ( + Configuration, + get_token_config_for_representation_and_collection, + get_token_permissions, +) from dump_things_service.auth import ( AuthenticationInfo, AuthenticationSource, InvalidTokenError, ) -from dump_things_service.abstract_config import ( - get_token_permissions, - get_token_config_for_representation_and_collection, -) class ConfigAuthenticationSource(AuthenticationSource): diff --git a/dump_things_service/auth/forgejo.py b/dump_things_service/auth/forgejo.py index 3b99524..b43ee58 100644 --- a/dump_things_service/auth/forgejo.py +++ b/dump_things_service/auth/forgejo.py @@ -12,8 +12,8 @@ from __future__ import annotations import hashlib import logging import time +from collections.abc import Callable from functools import wraps -from typing import Callable import requests from requests.exceptions import Timeout @@ -22,13 +22,13 @@ from dump_things_service import ( HTTP_300_MULTIPLE_CHOICES, HTTP_401_UNAUTHORIZED, ) +from dump_things_service.abstract_config import TokenPermission from dump_things_service.auth import ( AuthenticationError, AuthenticationInfo, AuthenticationSource, InvalidTokenError, ) -from dump_things_service.abstract_config import TokenPermission logger = logging.getLogger('dump_things_service') diff --git a/dump_things_service/authenticate.py b/dump_things_service/authenticate.py index cf4793b..519e7aa 100644 --- a/dump_things_service/authenticate.py +++ b/dump_things_service/authenticate.py @@ -1,48 +1,28 @@ from __future__ import annotations -import logging -from itertools import count from typing import TYPE_CHECKING from fastapi import ( - APIRouter, - Depends, - FastAPI, HTTPException, ) -from fastapi_pagination import ( - Page, - add_pagination, - paginate, -) from dump_things_service import ( HTTP_401_UNAUTHORIZED, - HTTP_404_NOT_FOUND, - HTTP_422_UNPROCESSABLE_CONTENT, abstract_config, ) from dump_things_service.abstract_config import ( check_collection, read_config, ) -from dump_things_service.api_key import api_key_header_scheme from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer -from dump_things_service.exceptions import CurieResolutionError from dump_things_service.instance_state import get_instance_state -from dump_things_service.lazy_list import ModifierList from dump_things_service.utils import ( authenticate_token, - check_bounds, - cleaned_json, - wrap_http_exception, ) if TYPE_CHECKING: - from pydantic import BaseModel from dump_things_service.backends import StorageBackend - from dump_things_service.lazy_list import LazyList from dump_things_service.store.model_store import _ModelStore diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 16e0258..a420fa3 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -7,10 +7,10 @@ The disk-layout is described in . from __future__ import annotations import logging +from collections.abc import Callable from pathlib import Path from typing import ( TYPE_CHECKING, - Callable, ) import yaml @@ -30,8 +30,8 @@ if TYPE_CHECKING: __all__ = [ - '_RecordDirStore', 'RecordDirStore', + '_RecordDirStore', ] ignored_files = {'.', '..', config_file_name} diff --git a/dump_things_service/backends/schema_type_layer.py b/dump_things_service/backends/schema_type_layer.py index 640519a..9dd4d18 100644 --- a/dump_things_service/backends/schema_type_layer.py +++ b/dump_things_service/backends/schema_type_layer.py @@ -34,8 +34,8 @@ if TYPE_CHECKING: __all__ = [ - '_SchemaTypeLayer', 'SchemaTypeLayer', + '_SchemaTypeLayer', ] @@ -96,8 +96,7 @@ class _SchemaTypeLayer(StorageBackend): # don't want to store it in the files. We add `schema_type` after # reading the record from disk. The value of `schema_type` is determined # by the class name of the record, which is stored in the path. - if 'schema_type' in json_object: - del json_object['schema_type'] + json_object.pop('schema_type', None) self.backend.add_record( iri=iri, class_name=class_name, diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index 5dd8523..b04fd06 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -62,8 +62,8 @@ if TYPE_CHECKING: __all__ = [ - '_SQLiteBackend', 'SQLiteBackend', + '_SQLiteBackend', ] logger = logging.getLogger('dump_things_service') diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 7907459..8517f7b 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -2,13 +2,19 @@ import logging import os import shutil from pathlib import Path -from typing import Any + +# This following lines are required for dynamic endpoint generation +from typing import ( + Annotated, # noqa: F401 -- used by autogenerated code + Any, +) from datalad_core.runners import ( - call_git_oneline, CommandError, + call_git_oneline, ) from fastapi import ( + Body, # noqa: F401 -- used by autogenerated code Depends, FastAPI, HTTPException, @@ -24,40 +30,49 @@ from starlette.responses import ( ) from dump_things_service import ( - Format, HTTP_400_BAD_REQUEST, HTTP_403_FORBIDDEN, HTTP_422_UNPROCESSABLE_CONTENT, + Format, ) from dump_things_service.abstract_config import ( CollectionConfig, - Configuration, ConfigAuthSpec, + Configuration, ForgejoAuthSpec, RecordDirBackendConfig, SQLiteBackendConfig, - read_config, check_collection, + read_config, +) +from dump_things_service.api_key import ( + api_key_header_scheme, ) from dump_things_service.audit.gitaudit import GitAuditBackend from dump_things_service.auth.config import ConfigAuthenticationSource from dump_things_service.auth.forgejo import ForgejoAuthenticationSource from dump_things_service.backends.record_dir_index import index_file_name from dump_things_service.backends.sqlite import record_file_name as sqlite_db_filename +from dump_things_service.converter import FormatConverter +from dump_things_service.curated import ( + store_curated_record, # noqa: F401 -- used by autogenerated code +) +from dump_things_service.exceptions import ( + ConfigCollisionError, + ConfigError, + CurieResolutionError, +) +from dump_things_service.incoming import ( + store_incoming_record, # noqa: F401 -- used by autogenerated code +) from dump_things_service.instance_state import ( InstanceState, InstanceStateCollectionInfo, - get_record_dir_config, get_instance_state, + get_record_dir_config, get_schema_info, record_dir_config_file_name, ) -from dump_things_service.converter import FormatConverter -from dump_things_service.exceptions import ( - ConfigError, - ConfigCollisionError, - CurieResolutionError, -) from dump_things_service.model import get_model_for_schema from dump_things_service.utils import ( combine_ttl, @@ -67,16 +82,9 @@ from dump_things_service.utils import ( var_escape, wrap_http_exception, ) - - -# This following lines are required for dynamic endpoint generation -from typing import Annotated # noqa: F401 -- used by autogenerated code -from fastapi import Body # noqa: F401 -- used by autogenerated code -from dump_things_service.api_key import api_key_header_scheme # noqa: F401 -- used by autogenerated code -from dump_things_service.curated import store_curated_record # noqa: F401 -- used by autogenerated code -from dump_things_service.incoming import store_incoming_record # noqa: F401 -- used by autogenerated code -from dump_things_service.validate import validate_record # noqa: F401 -- used by autogenerated code - +from dump_things_service.validate import ( + validate_record, # noqa: F401 -- used by autogenerated code +) logger = logging.getLogger('dump_things_service') @@ -191,7 +199,7 @@ def create_collection( audit_path.mkdir(parents=True) created_directories.append(audit_path) - except ConfigError as e: + except ConfigError: # Delete all directories that were created in this for directory in created_directories: shutil.rmtree(directory) @@ -368,7 +376,6 @@ def check_sqlite_compatibility( sqlite_db_path = Path(store_path / sqlite_db_filename) if not sqlite_db_path.exists(): raise ConfigError('No sqlite database found in existing store') - return def check_git_audit_compatibility( @@ -413,7 +420,7 @@ def create_endpoint( app: FastAPI, ): logger.info( - f'Creating %s-endpoints for collection: "%s"', + 'Creating %s-endpoints for collection: "%s"', operation_name, collection_name, ) diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index 5972996..494f9e7 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -22,19 +22,19 @@ from dump_things_service import ( reserved_collection_names, ) from dump_things_service.abstract_config import ( - Configuration, CollectionConfig, + Configuration, + get_config, + get_token_permissions, store_config, - get_config, get_token_permissions, ) from dump_things_service.admin import authenticate_admin from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.instance_state import get_instance_state, InstanceState -from dump_things_service.manifest import manifest_configuration from dump_things_service.exceptions import ConfigError +from dump_things_service.instance_state import InstanceState, get_instance_state +from dump_things_service.manifest import manifest_configuration from dump_things_service.utils import wrap_http_exception - logger = logging.getLogger('dump_things_service') router = APIRouter() diff --git a/dump_things_service/commands/copy_store.py b/dump_things_service/commands/copy_store.py index 8eeda94..27c2d61 100644 --- a/dump_things_service/commands/copy_store.py +++ b/dump_things_service/commands/copy_store.py @@ -5,6 +5,7 @@ from argparse import ArgumentParser from pathlib import Path from typing import TYPE_CHECKING +from dump_things_service.abstract_config import get_backend_and_extension from dump_things_service.backends.record_dir import ( RecordDirStore, _RecordDirStore, @@ -17,7 +18,6 @@ from dump_things_service.backends.sqlite import ( from dump_things_service.backends.sqlite import ( record_file_name as sqlite_record_file_name, ) -from dump_things_service.abstract_config import get_backend_and_extension if TYPE_CHECKING: from dump_things_service.backends import StorageBackend diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py index f63b098..2e5ff23 100644 --- a/dump_things_service/commands/download_config.py +++ b/dump_things_service/commands/download_config.py @@ -8,7 +8,6 @@ from argparse import ArgumentParser import requests import yaml - parser = ArgumentParser( prog='Download a complete configuration of a running service', description='Read a configuration from dump-things endpoints and create a ' diff --git a/dump_things_service/commands/gitaudit_rebuild_index.py b/dump_things_service/commands/gitaudit_rebuild_index.py index bde297e..5c885c8 100644 --- a/dump_things_service/commands/gitaudit_rebuild_index.py +++ b/dump_things_service/commands/gitaudit_rebuild_index.py @@ -6,7 +6,6 @@ from pathlib import Path from dump_things_service.audit.gitaudit import GitAuditBackend - parser = ArgumentParser( prog='Rebuild the index of a `gitaudit`-database', description='This command rebuilds the index of a `gitaudit`-database.' diff --git a/dump_things_service/commands/gitaudit_report.py b/dump_things_service/commands/gitaudit_report.py index 06600b8..cb92656 100644 --- a/dump_things_service/commands/gitaudit_report.py +++ b/dump_things_service/commands/gitaudit_report.py @@ -8,7 +8,6 @@ from pathlib import Path from dump_things_service.audit.gitaudit import GitAuditBackend - parser = ArgumentParser( prog='Report audit information for a PID', description='Report the audit information that was stored for a specific ' diff --git a/dump_things_service/commands/hash_token.py b/dump_things_service/commands/hash_token.py index 5f5d478..3d7312b 100644 --- a/dump_things_service/commands/hash_token.py +++ b/dump_things_service/commands/hash_token.py @@ -5,7 +5,6 @@ from argparse import ArgumentParser from dump_things_service.abstract_config import hash_token_representation - parser = ArgumentParser( prog='Hash a plain text token to create a hashed token in a dump-things server', description='Hash a token and print the calculated hash value. The hash value ' diff --git a/dump_things_service/commands/rebuild_index.py b/dump_things_service/commands/rebuild_index.py index 0825dc4..09846e2 100644 --- a/dump_things_service/commands/rebuild_index.py +++ b/dump_things_service/commands/rebuild_index.py @@ -7,9 +7,8 @@ from pathlib import Path import yaml from dump_things_service import config_file_name -from dump_things_service.backends.record_dir_index import RecordDirIndex from dump_things_service.abstract_config import RecordDirConfigFileContent - +from dump_things_service.backends.record_dir_index import RecordDirIndex parser = ArgumentParser( prog='Rebuild the index of a `record_dir`-store', diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index 71a2395..76fb2ae 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -12,7 +12,6 @@ import yaml from dump_things_service.instance_state import get_record_dir_config - parser = ArgumentParser( prog='Establish a configuration in a running service', description='Read a configuration from a dump-things configuration-file ' @@ -85,14 +84,13 @@ def main(): if arguments.old_format: configuration = convert_config_1_to_config_2(configuration, arguments.store) - else: - if arguments.store: - print( - 'Warning: ignoring `--store` option because `--old-format` ' - 'is not provided.', - file=sys.stderr, - flush=True, - ) + elif arguments.store: + print( + 'Warning: ignoring `--store` option because `--old-format` ' + 'is not provided.', + file=sys.stderr, + flush=True, + ) assert configuration['type'] == 'collections', '`type: collections` missing in config-file' assert configuration['version'] == 2, '`version: 2` missing in config-file' @@ -110,9 +108,7 @@ def main(): try: establish_configuration( configuration, - arguments.send_to[:-1] - if arguments.send_to.endswith('/') - else arguments.send_to, + arguments.send_to.removesuffix('/'), admin_token, ) return 0 diff --git a/dump_things_service/converter.py b/dump_things_service/converter.py index ed85a8d..7b8fbb6 100644 --- a/dump_things_service/converter.py +++ b/dump_things_service/converter.py @@ -1,19 +1,19 @@ from __future__ import annotations import re +from collections.abc import Callable from functools import cache from json import loads as json_loads from typing import ( TYPE_CHECKING, Any, - Callable, ) -from linkml_runtime import SchemaView from linkml.utils.datautils import ( get_dumper, get_loader, ) +from linkml_runtime import SchemaView from rdflib.term import ( URIRef, _toPythonMapping, @@ -29,7 +29,6 @@ from dump_things_service.model import ( ) from dump_things_service.utils import cleaned_json - if TYPE_CHECKING: from types import ModuleType diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index fef4529..e685d6b 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -1,13 +1,11 @@ from __future__ import annotations import logging -from itertools import count from typing import TYPE_CHECKING from fastapi import ( APIRouter, Depends, - FastAPI, HTTPException, ) from fastapi_pagination import ( @@ -19,7 +17,7 @@ from fastapi_pagination import ( from dump_things_service import ( HTTP_401_UNAUTHORIZED, HTTP_404_NOT_FOUND, - HTTP_422_UNPROCESSABLE_CONTENT, abstract_config, + HTTP_422_UNPROCESSABLE_CONTENT, ) from dump_things_service.abstract_config import ( check_collection, diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index 2d888cd..1459222 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -22,8 +22,8 @@ from dump_things_service import ( from dump_things_service.abstract_config import ( check_collection, check_label, - get_config_labels, get_config, + get_config_labels, ) from dump_things_service.api_key import api_key_header_scheme from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index 359a185..b82facc 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -2,12 +2,12 @@ from __future__ import annotations import dataclasses import logging +from collections.abc import Callable from functools import cache from pathlib import Path from types import ModuleType from typing import ( Any, - Callable, ) import yaml @@ -17,11 +17,10 @@ from pydantic import ValidationError from yaml.scanner import ScannerError from dump_things_service.abstract_config import ( - RecordDirConfigFileContent, MappingMethod, + RecordDirConfigFileContent, mapping_functions, ) - from dump_things_service.converter import get_conversion_objects from dump_things_service.exceptions import ConfigError from dump_things_service.model import ( @@ -30,7 +29,6 @@ from dump_things_service.model import ( get_schema_view, ) - logger = logging.getLogger('dump_things_service') record_dir_config_file_name = '.dumpthings.yaml' diff --git a/dump_things_service/lazy_list.py b/dump_things_service/lazy_list.py index 83eb0de..35b9bad 100644 --- a/dump_things_service/lazy_list.py +++ b/dump_things_service/lazy_list.py @@ -27,10 +27,9 @@ from abc import ( from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable from typing import ( Any, - Callable, ) diff --git a/dump_things_service/main.py b/dump_things_service/main.py index 68cc747..edfebfc 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -182,15 +182,14 @@ if not arguments.admin_token_hash: arguments.admin_token_hash = hash_token_representation( os.environ.get('DTS_ADMIN_TOKEN', ''), ) -else: - # Validate the hash token format - if not hash_matcher.match(arguments.admin_token_hash): - print( - 'Hashed admin token is not a 64-digits hex-number', - file=sys.stderr, - flush=True, - ) - sys.exit(1) +# Validate the hash token format +elif not hash_matcher.match(arguments.admin_token_hash): + print( + 'Hashed admin token is not a 64-digits hex-number', + file=sys.stderr, + flush=True, + ) + sys.exit(1) # Set the log level @@ -280,14 +279,13 @@ if not ( ): if arguments.config: config_file = arguments.config + elif arguments.ignore_default_config_file: + config_file = None else: - if arguments.ignore_default_config_file: + from dump_things_service import config_file_name + config_file = g_instance_state.store_path / config_file_name + if not config_file.exists(): config_file = None - else: - from dump_things_service import config_file_name - config_file = g_instance_state.store_path / config_file_name - if not config_file.exists(): - config_file = None if config_file: logger.info( @@ -368,7 +366,7 @@ async def maintenance( if api_key is None: raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f'Token required for this operation', + detail='Token required for this operation', ) collection = body.collection @@ -387,14 +385,13 @@ async def maintenance( ): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f'Curator permissions required for this operation', + detail='Curator permissions required for this operation', ) if active: g_instance_state.maintenance_mode.add(collection) else: g_instance_state.maintenance_mode.remove(collection) - return @app.get( diff --git a/dump_things_service/manifest.py b/dump_things_service/manifest.py index 71cb722..a1acc44 100644 --- a/dump_things_service/manifest.py +++ b/dump_things_service/manifest.py @@ -12,7 +12,6 @@ from dump_things_service.collection import ( ) from dump_things_service.instance_state import InstanceState - logger = logging.getLogger('dump_things_service') tag_groups = [ diff --git a/dump_things_service/store/model_store.py b/dump_things_service/store/model_store.py index f77be4e..787258b 100644 --- a/dump_things_service/store/model_store.py +++ b/dump_things_service/store/model_store.py @@ -17,8 +17,8 @@ if TYPE_CHECKING: from pydantic import BaseModel from dump_things_service.backends import ( - _RecordInfo, StorageBackend, + _RecordInfo, ) from dump_things_service.lazy_list import LazyList @@ -252,10 +252,9 @@ def ModelStore( # noqa: N802 # We store a pointer to the backend in the value to ensure that the # backend object exists while we use its `id` as a key. _existing_model_stores[id(backend)] = existing_model_store, backend - else: - # Check that the schemas are compatible, if the backend is reused. - if existing_model_store.schema != schema: - msg = 'Backend is already used in a ModelStore with a different schema' - raise ValueError(msg) + # Check that the schemas are compatible, if the backend is reused. + elif existing_model_store.schema != schema: + msg = 'Backend is already used in a ModelStore with a different schema' + raise ValueError(msg) return existing_model_store diff --git a/dump_things_service/tests/create_store.py b/dump_things_service/tests/create_store.py index 5f038c0..f5e0660 100644 --- a/dump_things_service/tests/create_store.py +++ b/dump_things_service/tests/create_store.py @@ -4,18 +4,19 @@ from typing import TYPE_CHECKING import yaml -from dump_things_service.backends.record_dir import RecordDirStore -from dump_things_service.backends.sqlite import ( - SQLiteBackend, - record_file_name as sqlite_record_file_name, -) from dump_things_service.abstract_config import ( - RecordDirBackendConfig, CollectionConfig, Configuration, MappingMethod, + RecordDirBackendConfig, mapping_functions, ) +from dump_things_service.backends.sqlite import ( + SQLiteBackend, +) +from dump_things_service.backends.sqlite import ( + record_file_name as sqlite_record_file_name, +) from dump_things_service.model import get_model_for_schema from dump_things_service.resolve_curie import resolve_curie diff --git a/dump_things_service/tests/fixtures.py b/dump_things_service/tests/fixtures.py index 1f19f86..127c4cd 100644 --- a/dump_things_service/tests/fixtures.py +++ b/dump_things_service/tests/fixtures.py @@ -11,20 +11,23 @@ import yaml from dump_things_service.abstract_config import ( GitAuditBackendConfig, SQLiteBackendConfig, + TagSpec, TokenCollectionConfig, - TokenModes, hash_token_representation, TagSpec, + TokenModes, + hash_token_representation, ) from dump_things_service.backends import StorageBackend from dump_things_service.backends.record_dir import RecordDirStore from dump_things_service.backends.sqlite import ( SQLiteBackend, +) +from dump_things_service.backends.sqlite import ( record_file_name as sqlite_db_filename, ) from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.instance_state import get_mapping_function_by_name from dump_things_service.model import get_model_for_schema from dump_things_service.resolve_curie import resolve_curie -from dump_things_service.token_endpoints import TokenRequest from dump_things_service.tests.create_store import ( pid, pid_curated, @@ -33,7 +36,7 @@ from dump_things_service.tests.create_store import ( test_record_curated, test_record_trr, ) - +from dump_things_service.token_endpoints import TokenRequest # String representation of curated- and incoming-path curated = 'curated' @@ -64,7 +67,7 @@ g_default_collections[6].submission_tags = TagSpec( g_default_collections.append( CollectionRequest( - name=f'collection_8', + name='collection_8', default_token='test_default_token', curated=PurePosixPath(f'{curated}/collection_8'), schema=test_schema_location, @@ -152,7 +155,7 @@ g_default_tokens = [ hashed=False, representation='token-2', collections={ - f'collection_2': TokenCollectionConfig( + 'collection_2': TokenCollectionConfig( mode=TokenModes.WRITE_COLLECTION, incoming_label='in_token-2', ) @@ -164,7 +167,7 @@ g_default_tokens = [ hashed=False, representation='token-8', collections={ - f'collection_8': TokenCollectionConfig( + 'collection_8': TokenCollectionConfig( mode=TokenModes.WRITE_COLLECTION, incoming_label='test_user_8', ) diff --git a/dump_things_service/tests/test_basic.py b/dump_things_service/tests/test_basic.py index 7591678..7d3e6ea 100644 --- a/dump_things_service/tests/test_basic.py +++ b/dump_things_service/tests/test_basic.py @@ -1,7 +1,5 @@ -import pytest # F401 -from . import schema_file from .. import ( HTTP_200_OK, HTTP_400_BAD_REQUEST, @@ -11,14 +9,13 @@ from .. import ( HTTP_503_SERVICE_UNAVAILABLE, ) from ..__about__ import __version__ -from ..utils import cleaned_json +from . import schema_file from .create_store import ( given_name, pid, ) from .test_utils import basic_write_locations - extra_record = { 'schema_type': 'abc:Person', 'pid': 'abc:aaaa', diff --git a/dump_things_service/tests/test_collection_administration.py b/dump_things_service/tests/test_collection_administration.py index 5dc2f39..3e2f41c 100644 --- a/dump_things_service/tests/test_collection_administration.py +++ b/dump_things_service/tests/test_collection_administration.py @@ -6,10 +6,10 @@ from pathlib import ( from starlette.testclient import TestClient from dump_things_service import ( - HTTP_201_CREATED, HTTP_200_OK, - HTTP_404_NOT_FOUND, + HTTP_201_CREATED, HTTP_401_UNAUTHORIZED, + HTTP_404_NOT_FOUND, ) from dump_things_service.abstract_config import ( GitAuditBackendConfig, @@ -19,10 +19,9 @@ from dump_things_service.abstract_config import ( ) from dump_things_service.collection_endpoints import CollectionRequest from dump_things_service.token_endpoints import ( - TokenRequest, AdminTokenRequest, + TokenRequest, ) -from dump_things_service.utils import cleaned_json # String representation of curated- and incoming-path curated = 'admin_test_curated' @@ -259,7 +258,7 @@ def test_collection_reading(fastapi_client_simple): # Check that the new admin token is not yet working response = test_client.get( - f'/collections', + '/collections', headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == HTTP_200_OK @@ -273,7 +272,7 @@ def test_admin_token_management(fastapi_client_simple): # Check that the new admin token is not yet working response = test_client.get( - f'/collections/collection_1', + '/collections/collection_1', headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_401_UNAUTHORIZED @@ -288,14 +287,14 @@ def test_admin_token_management(fastapi_client_simple): # Try the new token response = test_client.get( - f'/collections/collection_1', + '/collections/collection_1', headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_200_OK # Check that the token shows up in the token list response = test_client.get( - f'/admin_tokens', + '/admin_tokens', headers={'x-dumpthings-token': plain_new_admin_token}, ) assert response.status_code == HTTP_200_OK @@ -310,7 +309,7 @@ def test_admin_token_management(fastapi_client_simple): assert response.status_code == HTTP_200_OK response = test_client.get( - f'/admin_tokens', + '/admin_tokens', headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == HTTP_200_OK diff --git a/dump_things_service/tests/test_config.py b/dump_things_service/tests/test_config.py index 72efe9c..b2c36ef 100644 --- a/dump_things_service/tests/test_config.py +++ b/dump_things_service/tests/test_config.py @@ -22,13 +22,12 @@ from dump_things_service.exceptions import ConfigError from dump_things_service.tests import schema_file from dump_things_service.token_endpoints import TokenRequest - collection_request_pattern = CollectionRequest( name='', schema=str(schema_file), default_token='test_default_token', curated=PurePosixPath('curate_dir'), - incoming=PurePosixPath(f'incoming_dir'), + incoming=PurePosixPath('incoming_dir'), ) @@ -42,7 +41,7 @@ def test_illegal_collection_name_detection(fastapi_client_simple): dump_things_private_collection_name, ): response = test_client.post( - f'/collections', + '/collections', json={ **collection_request_pattern.model_dump(mode='json', by_alias=True), 'name': name, @@ -62,7 +61,7 @@ def test_collection_dir_reuse_detection(fastapi_client_simple): ('curated/collection_1', 'incoming/collection_2'), ): response = test_client.post( - f'/collections', + '/collections', json={ **collection_request_pattern.model_dump(mode='json', by_alias=True), 'curated': curated_path, diff --git a/dump_things_service/tests/test_curated.py b/dump_things_service/tests/test_curated.py index 47737a8..1c34f04 100644 --- a/dump_things_service/tests/test_curated.py +++ b/dump_things_service/tests/test_curated.py @@ -1,17 +1,17 @@ from __future__ import annotations -import pytest import time -import yaml from itertools import count +import pytest +import yaml + from dump_things_service import ( HTTP_200_OK, HTTP_404_NOT_FOUND, ) from dump_things_service.instance_state import get_instance_state - delete_record = { 'schema_type': 'abc:Person', 'pid': 'abc:delete-me', @@ -185,5 +185,5 @@ def test_audit_backend_auto_flush(fastapi_client_simple): break i += 1 if i == 10: - raise ValueError(f'auto flush did not trigger within 10 seconds') + raise ValueError('auto flush did not trigger within 10 seconds') time.sleep(1) diff --git a/dump_things_service/tests/test_ifabsent_patch.py b/dump_things_service/tests/test_ifabsent_patch.py index 95e855b..5146f01 100644 --- a/dump_things_service/tests/test_ifabsent_patch.py +++ b/dump_things_service/tests/test_ifabsent_patch.py @@ -6,7 +6,6 @@ import linkml.generators.common.ifabsent_processor as if_abs_proc import dump_things_service.patches.ifabsent_processing - # Path to a local simple test schema schema_dir = Path(__file__).parent / 'assets' diff --git a/dump_things_service/tests/test_roundtrip_flatsocial.py b/dump_things_service/tests/test_roundtrip_flatsocial.py index ada9aeb..83ac558 100644 --- a/dump_things_service/tests/test_roundtrip_flatsocial.py +++ b/dump_things_service/tests/test_roundtrip_flatsocial.py @@ -1,6 +1,5 @@ -import pytest # noqa: F401 - import freezegun +import pytest # noqa: F401 from .. import HTTP_200_OK from ..utils import cleaned_json diff --git a/dump_things_service/tests/test_unicode.py b/dump_things_service/tests/test_unicode.py index 68ce9fe..5a1deb6 100644 --- a/dump_things_service/tests/test_unicode.py +++ b/dump_things_service/tests/test_unicode.py @@ -2,7 +2,6 @@ from pathlib import Path from .. import HTTP_200_OK - # Path to a local simple test schema schema_file = Path(__file__).parent / 'testschema.yaml' diff --git a/dump_things_service/token_endpoints.py b/dump_things_service/token_endpoints.py index 06d2d0f..a45e1f2 100644 --- a/dump_things_service/token_endpoints.py +++ b/dump_things_service/token_endpoints.py @@ -30,12 +30,11 @@ from dump_things_service.abstract_config import ( ) from dump_things_service.admin import authenticate_admin from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.instance_state import get_instance_state from dump_things_service.exceptions import ConfigError +from dump_things_service.instance_state import get_instance_state from dump_things_service.manifest import manifest_configuration from dump_things_service.utils import wrap_http_exception - logger = logging.getLogger('dump_things_service') router = APIRouter() @@ -154,7 +153,7 @@ def create_or_replace_token( token_representation=body.representation, ) if existing_token_info: - detail= f"Token with identical representation already exists." + detail= "Token with identical representation already exists." raise HTTPException(status_code=HTTP_409_CONFLICT, detail=detail) else: # Generate a random representation that does not yet exist. diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index ebb7d69..f40c903 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -31,12 +31,10 @@ from dump_things_service.abstract_config import ( Configuration, TokenModes, TokenPermission, - mode_mapping, check_collection, get_collection_config_by_name, - get_default_token_config, get_mapping_function_by_name, - get_token_config_for_representation_and_collection, + mode_mapping, ) from dump_things_service.auth import ( AuthenticationError, @@ -83,7 +81,7 @@ def cleaned_json(data: JSON, remove_keys: tuple[str, ...] = ('@type',)) -> JSON: return { key: cleaned_json(value, remove_keys) for key, value in data.items() - if key not in remove_keys and data[key] is not None + if key not in remove_keys and value is not None } return data @@ -323,8 +321,8 @@ def create_token_store( collection_name: str, store_dir: Path, ) -> _ModelStore: - from dump_things_service.backends.schema_type_layer import SchemaTypeLayer from dump_things_service.abstract_config import get_backend_and_extension + from dump_things_service.backends.schema_type_layer import SchemaTypeLayer from dump_things_service.exceptions import ConfigError from dump_things_service.store.model_store import ModelStore @@ -394,8 +392,8 @@ def create_record_dir_token_store_backend( mapping_function: str, suffix: str, ) -> _RecordDirStore: - from dump_things_service.instance_state import record_dir_config_file_name from dump_things_service.backends.record_dir import RecordDirStore + from dump_things_service.instance_state import record_dir_config_file_name # Write the configuration to the store, if it does not yet exist. if not (store_dir / record_dir_config_file_name).exists(): -- 2.52.0 From c332abbd1b432894fee88737c538c3516a741270 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 20:41:05 +0200 Subject: [PATCH 16/19] chore: auto-apply code formatting fixes via `hatch check fmt --fix` --- dump_things_service/abstract_config.py | 97 +++++---- dump_things_service/admin.py | 6 +- dump_things_service/audit/__init__.py | 12 +- dump_things_service/audit/gitaudit.py | 136 +++++++------ .../audit/tests/test_gitaudit.py | 15 +- dump_things_service/auth/__init__.py | 1 + dump_things_service/auth/config.py | 13 +- dump_things_service/auth/forgejo.py | 39 ++-- dump_things_service/authenticate.py | 6 +- dump_things_service/backends/__init__.py | 50 +++-- dump_things_service/backends/record_dir.py | 68 ++++--- .../backends/record_dir_index.py | 62 +++--- .../backends/schema_type_layer.py | 62 +++--- dump_things_service/backends/sqlite.py | 85 ++++---- .../backends/tests/test_record_dir.py | 4 +- dump_things_service/collection.py | 187 +++++++++++------- dump_things_service/collection_endpoints.py | 50 +++-- dump_things_service/commands/check_pids.py | 10 +- .../commands/create_merged_schema.py | 13 +- .../commands/download_config.py | 58 +++--- .../commands/gitaudit_rebuild_index.py | 5 +- .../commands/gitaudit_report.py | 8 +- dump_things_service/commands/hash_token.py | 5 +- dump_things_service/commands/upload_config.py | 104 +++++----- dump_things_service/converter.py | 10 +- dump_things_service/curated.py | 43 ++-- dump_things_service/incoming.py | 72 +++---- dump_things_service/instance_state.py | 16 +- dump_things_service/lazy_list.py | 2 +- dump_things_service/main.py | 111 ++++++----- dump_things_service/manifest.py | 24 ++- dump_things_service/model.py | 8 +- dump_things_service/store/model_store.py | 78 ++++---- dump_things_service/tests/fixtures.py | 77 ++++---- dump_things_service/tests/test_auth.py | 22 ++- dump_things_service/tests/test_basic.py | 5 +- .../tests/test_collection_administration.py | 14 +- dump_things_service/tests/test_config.py | 44 +++-- dump_things_service/tests/test_curated.py | 2 + .../tests/test_extract_inline.py | 31 +-- .../tests/test_ifabsent_patch.py | 9 +- dump_things_service/tests/test_incoming.py | 39 ++-- .../tests/test_roundtrip_flatsocial.py | 12 +- .../tests/test_token_endpoints.py | 7 +- dump_things_service/tests/test_unicode.py | 6 +- dump_things_service/tests/test_validate.py | 6 +- .../tests/test_web_interface.py | 30 +-- dump_things_service/token_endpoints.py | 75 +++---- dump_things_service/utils.py | 159 ++++++++------- dump_things_service/validate.py | 33 ++-- 50 files changed, 1058 insertions(+), 973 deletions(-) diff --git a/dump_things_service/abstract_config.py b/dump_things_service/abstract_config.py index b600cf0..cfca7f0 100644 --- a/dump_things_service/abstract_config.py +++ b/dump_things_service/abstract_config.py @@ -102,7 +102,9 @@ class CollectionConfig(BaseModel): curated: PurePosixPath schema_location: str = Field(alias='schema') incoming: PurePosixPath | None = None - backend: RecordDirBackendConfig | SQLiteBackendConfig = RecordDirBackendConfig(type='record_dir+stl') + backend: RecordDirBackendConfig | SQLiteBackendConfig = RecordDirBackendConfig( + type='record_dir+stl' + ) auth_sources: list[ForgejoAuthSpec | ConfigAuthSpec] = [ConfigAuthSpec()] audit_backends: list[GitAuditBackendConfig] = [] submission_tags: TagSpec = TagSpec() @@ -199,7 +201,7 @@ def get_token_permissions(mode: str) -> TokenPermission: def get_config_backends( - store_path: Path, + store_path: Path, ) -> tuple[_RecordDirStore, GitAuditBackend]: global config_audit global config_backend @@ -210,9 +212,7 @@ def get_config_backends( if config_backend is None: config_backend = RecordDirStore( - config_path, - mapping_functions[MappingMethod.digest_md5], - 'yaml' + config_path, mapping_functions[MappingMethod.digest_md5], 'yaml' ) audit_path = store_path / config_audit_path @@ -225,8 +225,8 @@ def get_config_backends( def read_config( - store_path: Path, - force_reload: bool = False, + store_path: Path, + force_reload: bool = False, ) -> Configuration: global g_abstract_configuration @@ -243,7 +243,7 @@ def read_config( if record_info else Configuration( type='collections', - version = 2, + version=2, ) ) except ValidationError as ve: @@ -262,8 +262,8 @@ def get_config() -> Configuration: def store_config( - store_path, - config: Configuration, + store_path, + config: Configuration, ): global g_abstract_configuration @@ -273,7 +273,7 @@ def store_config( config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object=json_object + json_object=json_object, ) audit_backend.add_record( record=json_object, @@ -283,8 +283,8 @@ def store_config( def tokens_for_collection( - config: Configuration, - collection: str, + config: Configuration, + collection: str, ) -> Iterable[TokenConfig]: yield from ( token @@ -294,8 +294,8 @@ def tokens_for_collection( def check_collection( - abstract_config: Configuration, - collection: str, + abstract_config: Configuration, + collection: str, ): if collection not in abstract_config.collections: raise HTTPException( @@ -305,18 +305,17 @@ def check_collection( def check_label( - store_path: Path, - abstract_config: Configuration, - collection: str, - label: str, + store_path: Path, + abstract_config: Configuration, + collection: str, + label: str, ): from dump_things_service.utils import get_on_disk_labels """Check that a label exists in a collection configuration or on disk""" - if ( - label not in get_config_labels(abstract_config, collection) - and label not in get_on_disk_labels(store_path, abstract_config, collection) - ): + if label not in get_config_labels( + abstract_config, collection + ) and label not in get_on_disk_labels(store_path, abstract_config, collection): raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"No incoming label: '{label}' in collection: '{collection}'.", @@ -324,8 +323,8 @@ def check_label( def get_config_labels( - abstract_config: Configuration, - collection: str, + abstract_config: Configuration, + collection: str, ) -> set[str]: check_collection(abstract_config, collection) return { @@ -335,17 +334,14 @@ def get_config_labels( } -def get_default_token_name( - abstract_config: Configuration, - collection: str -) -> str: +def get_default_token_name(abstract_config: Configuration, collection: str) -> str: check_collection(abstract_config, collection) return abstract_config.collections[collection].default_token def get_token_info_by_representation( - abstract_config: Configuration, - token_representation: str, + abstract_config: Configuration, + token_representation: str, ) -> tuple[str, TokenConfig] | None: """Get the name of the token given in `token_representation`""" hashed_representation = hash_token_representation(token_representation) @@ -360,23 +356,22 @@ def get_token_info_by_representation( def hash_token_representation( - token_representation: str, + token_representation: str, ) -> str: return hashlib.sha256(token_representation.encode()).hexdigest() def get_token_config_by_name( - abstract_config: Configuration, - token_name: str, + abstract_config: Configuration, + token_name: str, ) -> TokenConfig | None: return abstract_config.tokens.get(token_name) def get_token_infos_for_collection( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> Iterable[tuple[str, TokenConfig, TokenCollectionConfig]]: - yield from { (token_name, token_config, token_collection_config) for token_name, token_config in abstract_config.tokens.items() @@ -386,11 +381,10 @@ def get_token_infos_for_collection( def get_token_config_for_representation_and_collection( - abstract_config: Configuration, - collection_name: str, - token_representation: str, + abstract_config: Configuration, + collection_name: str, + token_representation: str, ) -> tuple[str, TokenConfig, TokenCollectionConfig] | None: - token_info = get_token_info_by_representation( abstract_config=abstract_config, token_representation=token_representation, @@ -404,8 +398,8 @@ def get_token_config_for_representation_and_collection( def get_collection_config_by_name( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> CollectionConfig: collection_config = abstract_config.collections.get(collection_name) if not collection_config: @@ -417,10 +411,9 @@ def get_collection_config_by_name( def get_default_token_config( - abstract_config: Configuration, - collection: str, + abstract_config: Configuration, + collection: str, ) -> TokenConfig | None: - default_token_name = get_collection_config_by_name( abstract_config, collection, @@ -444,18 +437,18 @@ def get_hex_digest(hasher: Callable, data: str) -> str: def mapping_digest_p3( - hasher: Callable, - pid: str, - suffix: str, + hasher: Callable, + pid: str, + suffix: str, ) -> Path: hex_digest = get_hex_digest(hasher, pid) return Path(hex_digest[:3]) / (hex_digest[3:] + '.' + suffix) def mapping_digest_p3_p3( - hasher: Callable, - pid: str, - suffix: str, + hasher: Callable, + pid: str, + suffix: str, ) -> Path: hex_digest = get_hex_digest(hasher, pid) return Path(hex_digest[:3]) / hex_digest[3:6] / (hex_digest[6:] + '.' + suffix) diff --git a/dump_things_service/admin.py b/dump_things_service/admin.py index 201e4d9..7f94ee8 100644 --- a/dump_things_service/admin.py +++ b/dump_things_service/admin.py @@ -13,9 +13,9 @@ logger = logging.getLogger('dump_things_service') def authenticate_admin( - instance_state: InstanceState, - abstract_config: Configuration, - api_key: str, + instance_state: InstanceState, + abstract_config: Configuration, + api_key: str, ): if api_key: hashed_token_representation = hash_token_representation(api_key) diff --git a/dump_things_service/audit/__init__.py b/dump_things_service/audit/__init__.py index 58b8458..68f671f 100644 --- a/dump_things_service/audit/__init__.py +++ b/dump_things_service/audit/__init__.py @@ -7,10 +7,10 @@ from abc import ( class AuditBackend(metaclass=ABCMeta): @abstractmethod def add_record( - self, - record: dict, - committer_id: str, - author_id: str | None = None, + self, + record: dict, + committer_id: str, + author_id: str | None = None, ) -> None: """Add information about a new record version to the audit log @@ -35,8 +35,8 @@ class AuditBackend(metaclass=ABCMeta): @abstractmethod def get_audit_log( - self, - record_id: str, + self, + record_id: str, ) -> dict: """Get the content of the audit log diff --git a/dump_things_service/audit/gitaudit.py b/dump_things_service/audit/gitaudit.py index fec83dc..8bf7123 100644 --- a/dump_things_service/audit/gitaudit.py +++ b/dump_things_service/audit/gitaudit.py @@ -6,6 +6,7 @@ committed. Changes are annotated with a time stamp and a user-id """ + from __future__ import annotations import hashlib @@ -34,9 +35,9 @@ index_file_name = 'gitaudit_index.log' class FlushingThread(Thread): def __init__( - self, - backend: GitAuditBackend, - auto_flush_timeout: int, + self, + backend: GitAuditBackend, + auto_flush_timeout: int, ): super().__init__() self.auto_flush_timeout = auto_flush_timeout @@ -55,11 +56,10 @@ class FlushingThread(Thread): class GitAuditBackend(AuditBackend): - def __init__( - self, - path: Path, - auto_flush_timeout: int = 60, + self, + path: Path, + auto_flush_timeout: int = 60, ): self.path = path self.index_path = None @@ -81,10 +81,10 @@ class GitAuditBackend(AuditBackend): self.flushing_thread = None def add_record( - self, - record: dict, - committer_id: str, - author_id: str | None = None, + self, + record: dict, + committer_id: str, + author_id: str | None = None, ) -> None: with self.lock: author_id = committer_id if author_id is None else author_id @@ -108,15 +108,15 @@ class GitAuditBackend(AuditBackend): self.last_flush_time = time.time() def get_audit_log( - self, - record_id: str, + self, + record_id: str, ) -> dict: with self.lock: return self._locked_get_audit_log(record_id) def _locked_get_audit_log( - self, - record_id: str, + self, + record_id: str, ) -> dict: self._locked_flush() @@ -124,17 +124,25 @@ class GitAuditBackend(AuditBackend): # the records changes = [] yaml_location, log_location = map(str, self._get_location_for(record_id)[1:]) - commit_hashes = call_git( - ['log', '--format=%H', '--', log_location], - cwd=self.path, - capture_output=True, - ).decode().splitlines() - for commit_hash in commit_hashes: - log_diff_lines = call_git( - ['show', '--format=%b', commit_hash, '--', log_location], + commit_hashes = ( + call_git( + ['log', '--format=%H', '--', log_location], cwd=self.path, capture_output=True, - ).decode().splitlines() + ) + .decode() + .splitlines() + ) + for commit_hash in commit_hashes: + log_diff_lines = ( + call_git( + ['show', '--format=%b', commit_hash, '--', log_location], + cwd=self.path, + capture_output=True, + ) + .decode() + .splitlines() + ) # Get the log entry log_line = tuple( filter( @@ -145,11 +153,15 @@ class GitAuditBackend(AuditBackend): log_entry = json.loads(log_line) # Get the YAML diff - yaml_diff_lines = call_git( - ['show', '--format=%b', commit_hash, '--', yaml_location], - cwd=self.path, - capture_output=True, - ).decode().splitlines() + yaml_diff_lines = ( + call_git( + ['show', '--format=%b', commit_hash, '--', yaml_location], + cwd=self.path, + capture_output=True, + ) + .decode() + .splitlines() + ) yaml_diff = '\n'.join(filter(lambda l: l != '', yaml_diff_lines)) + '\n' # Get the YAML content @@ -172,15 +184,15 @@ class GitAuditBackend(AuditBackend): return {c[0]: c[1:] for c in changes} def get_audit_logs( - self, - record_id_pattern: str, + self, + record_id_pattern: str, ) -> dict: with self.lock: return self._locked_get_audit_logs(record_id_pattern) def _locked_get_audit_logs( - self, - record_id_pattern: str, + self, + record_id_pattern: str, ) -> dict: self._locked_flush() matcher = re.compile(record_id_pattern) @@ -196,12 +208,12 @@ class GitAuditBackend(AuditBackend): } def _add_elements( - self, - record_id: str, - location: tuple[str, Path, Path], - committer_id: str, - author_id: str, - record: dict, + self, + record_id: str, + location: tuple[str, Path, Path], + committer_id: str, + author_id: str, + record: dict, ) -> bool: existing_record = self._read_record_from_repo_path(location[1]) if existing_record != record: @@ -217,10 +229,10 @@ class GitAuditBackend(AuditBackend): return False def _add_log_entry( - self, - log_location: Path, - committer_id: str, - author_id: str, + self, + log_location: Path, + committer_id: str, + author_id: str, ) -> None: time_stamp = datetime.now().isoformat() entry = { @@ -233,16 +245,16 @@ class GitAuditBackend(AuditBackend): self.current_change_set[log_location] = log_content def _add_index_entry( - self, - record_id: str, + self, + record_id: str, ): if record_id not in self.index: self.cached_index_entries.append(record_id) self.index.add(record_id) def _read_from_repo_path( - self, - path: Path, + self, + path: Path, ) -> bytes: try: return call_git( @@ -256,14 +268,14 @@ class GitAuditBackend(AuditBackend): raise def _read_record_from_repo_path( - self, - path: Path, + self, + path: Path, ): return yaml.safe_load(self._read_from_repo_path(path)) def _has_pending_changes( - self, - location: tuple[str, Path, Path], + self, + location: tuple[str, Path, Path], ) -> bool: log_pending = location[1] in self.current_change_set record_pending = location[2] in self.current_change_set @@ -285,8 +297,8 @@ class GitAuditBackend(AuditBackend): self.current_change_set = {} def _get_location_for( - self, - record_id: str, + self, + record_id: str, ) -> tuple[str, Path, Path]: base = hashlib.sha1(record_id.encode()).hexdigest() dir_1, dir_2, name = base[0:3], base[3:6], base[6:] @@ -324,19 +336,23 @@ class GitAuditBackend(AuditBackend): self.index = set(line.strip() for line in f) def _add_to_index( - self, - record_id: str, + self, + record_id: str, ): if record_id not in self.index: self.cached_index_entries.append(record_id) self.index.add(record_id) def _rebuild_index(self): - tree_entries = call_git( - ['ls-tree', '-r', 'master:'], - cwd=self.path, - capture_output=True, - ).decode().splitlines() + tree_entries = ( + call_git( + ['ls-tree', '-r', 'master:'], + cwd=self.path, + capture_output=True, + ) + .decode() + .splitlines() + ) with open(self.index_path, 'w') as f: for line in tree_entries: if not line.endswith('.yaml'): diff --git a/dump_things_service/audit/tests/test_gitaudit.py b/dump_things_service/audit/tests/test_gitaudit.py index ef1e059..25ddf12 100644 --- a/dump_things_service/audit/tests/test_gitaudit.py +++ b/dump_things_service/audit/tests/test_gitaudit.py @@ -19,7 +19,7 @@ def _get_audit_log_lines(backend: GitAuditBackend, record_id: str) -> list[str]: def test_gitaudit_basic(tmp_path_factory): - tmp_path = tmp_path_factory.mktemp("gitaudit_backend") + tmp_path = tmp_path_factory.mktemp('gitaudit_backend') backend = GitAuditBackend(tmp_path) @@ -45,13 +45,12 @@ def test_gitaudit_basic(tmp_path_factory): changes = backend.get_audit_log(record_id) assert len(changes) == 4 assert tuple(map(lambda e: e[0:2], changes.values())) == tuple( - (f'committer_{100 + i}@x.org', f'author_{i}@y.org') - for i in range(4) + (f'committer_{100 + i}@x.org', f'author_{i}@y.org') for i in range(4) ) def test_gitaudit_identical_change(tmp_path_factory): - tmp_path = tmp_path_factory.mktemp("gitaudit_backend") + tmp_path = tmp_path_factory.mktemp('gitaudit_backend') backend = GitAuditBackend(tmp_path) @@ -59,13 +58,13 @@ def test_gitaudit_identical_change(tmp_path_factory): backend.add_record( record={'pid': record_id}, committer_id='committer_b@x.org', - author_id = 'author_b@y.org', + author_id='author_b@y.org', ) backend.add_record( record={'pid': record_id}, committer_id='committer_b@x.org', - author_id = 'author_b@y.org', + author_id='author_b@y.org', ) # Check that there is only one entry in the audit log @@ -83,7 +82,7 @@ def test_gitaudit_identical_change(tmp_path_factory): def test_gitaudit_huge_log(tmp_path_factory): - tmp_path = tmp_path_factory.mktemp("gitaudit_backend") + tmp_path = tmp_path_factory.mktemp('gitaudit_backend') backend = GitAuditBackend(tmp_path) @@ -96,7 +95,7 @@ def test_gitaudit_huge_log(tmp_path_factory): backend.add_record( record={'pid': record_id, 'content': f'j:{j}, i:{i}'}, committer_id='committer@x.org', - author_id = 'author@y.org', + author_id='author@y.org', ) # Check that the changes are reported diff --git a/dump_things_service/auth/__init__.py b/dump_things_service/auth/__init__.py index 051720f..00ff138 100644 --- a/dump_things_service/auth/__init__.py +++ b/dump_things_service/auth/__init__.py @@ -8,6 +8,7 @@ determine: - the incoming_label to be used with the token """ + from __future__ import annotations import abc diff --git a/dump_things_service/auth/config.py b/dump_things_service/auth/config.py index 7aeb45c..1808214 100644 --- a/dump_things_service/auth/config.py +++ b/dump_things_service/auth/config.py @@ -1,4 +1,4 @@ -"""Use configuration information to fetch token permissions, ids, and incoming_label """ +"""Use configuration information to fetch token permissions, ids, and incoming_label""" from dump_things_service.abstract_config import ( Configuration, @@ -14,18 +14,17 @@ from dump_things_service.auth import ( class ConfigAuthenticationSource(AuthenticationSource): def __init__( - self, - abstract_configuration: Configuration, - collection_name: str, + self, + abstract_configuration: Configuration, + collection_name: str, ): self.abstract_configuration = abstract_configuration self.collection_name = collection_name def authenticate( - self, - token_representation: str, + self, + token_representation: str, ) -> AuthenticationInfo: - result = get_token_config_for_representation_and_collection( self.abstract_configuration, self.collection_name, diff --git a/dump_things_service/auth/forgejo.py b/dump_things_service/auth/forgejo.py index b43ee58..6288192 100644 --- a/dump_things_service/auth/forgejo.py +++ b/dump_things_service/auth/forgejo.py @@ -7,6 +7,7 @@ Note: for some reason, the request: does not require a token. If the owner and the repo are known, the request will emit a complete repository-record including the complete owner-record. """ + from __future__ import annotations import hashlib @@ -46,7 +47,8 @@ class MethodCache: def cache_temporary( duration: int = 300, ) -> Callable: - """ Cache results for a given time (default: 300 seconds) """ + """Cache results for a given time (default: 300 seconds)""" + def decorator(func: Callable) -> Callable: @wraps(func) def wrapper(*args, **kwargs): @@ -56,12 +58,15 @@ class MethodCache: if cached_data is None or time.time() - cached_data[0] > duration: self.__cached_data[key] = (time.time(), func(*args, **kwargs)) return self.__cached_data[key][1] + return wrapper + return decorator class RemoteAuthenticationError(AuthenticationError): """Exception for remote authentication errors.""" + def __init__(self, status: int, message: str): self.status = status self.message = message @@ -139,8 +144,8 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): @MethodCache.cache_temporary(duration=120) def _get_user( - self, - token: str, + self, + token: str, ) -> dict: return self._get_json_from_endpoint('user', token) @@ -183,8 +188,8 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): @staticmethod def _get_permissions( - code_permission: str, - action_permission: str, + code_permission: str, + action_permission: str, ) -> TokenPermission: is_curator = action_permission == 'write' read = code_permission in ('read', 'write') or is_curator @@ -197,11 +202,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): zones_access=is_curator, ) - def _get_unit_content( - self, - team: dict, - unit_name: str - ) -> str: + def _get_unit_content(self, team: dict, unit_name: str) -> str: permissions = team['units_map'].get(unit_name) if not permissions: logger.debug(f'no unit `repo.actions` in team {self.team}') @@ -216,23 +217,22 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): return permissions def _instance_label(self) -> str: - return self.instance_id or hashlib.md5( - self.api_url.encode() - ).hexdigest() + return self.instance_id or hashlib.md5(self.api_url.encode()).hexdigest() @MethodCache.cache_temporary(duration=60) def authenticate( self, token: str, ) -> AuthenticationInfo: - - logger.debug(f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}') + logger.debug( + f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}' + ) user_teams = self._get_teams_for_user(token) logger.debug(f'user_teams: {user_teams}') if self.team not in user_teams: - logger.debug(f'{self.team} not in user\'s teams') + logger.debug(f"{self.team} not in user's teams") msg = f'token user is not member of team `{self.team}`' raise RemoteAuthenticationError( status=HTTP_401_UNAUTHORIZED, @@ -281,8 +281,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): action_permissions, ), user_id=user_info['email'], - incoming_label= - f'forgejo-{self._instance_label()}-team-{organization["name"]}-{team["name"]}' - if self.label_type == 'team' - else f'forgejo-{self._instance_label()}-user-{user_info["login"]}', + incoming_label=f'forgejo-{self._instance_label()}-team-{organization["name"]}-{team["name"]}' + if self.label_type == 'team' + else f'forgejo-{self._instance_label()}-user-{user_info["login"]}', ) diff --git a/dump_things_service/authenticate.py b/dump_things_service/authenticate.py index 519e7aa..1ecb585 100644 --- a/dump_things_service/authenticate.py +++ b/dump_things_service/authenticate.py @@ -21,16 +21,14 @@ from dump_things_service.utils import ( ) if TYPE_CHECKING: - from dump_things_service.backends import StorageBackend from dump_things_service.store.model_store import _ModelStore def get_store_and_backend( - collection: str, - plain_token: str | None, + collection: str, + plain_token: str | None, ) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]: - # A token is required if plain_token is None: raise HTTPException( diff --git a/dump_things_service/backends/__init__.py b/dump_things_service/backends/__init__.py index 901b9e8..74128cb 100644 --- a/dump_things_service/backends/__init__.py +++ b/dump_things_service/backends/__init__.py @@ -83,12 +83,12 @@ class BackendResultList(LazyList): @abstractmethod def generate_result( - self, - index: int, - iri: str, - class_name: str, - sort_key: str, - private: Any, + self, + index: int, + iri: str, + class_name: str, + sort_key: str, + private: Any, ) -> RecordInfo: """ Generate a record info object from the provided parameters. @@ -105,23 +105,21 @@ class BackendResultList(LazyList): class StorageBackend(metaclass=ABCMeta): def __init__( - self, - order_by: Iterable[str] | None = None, + self, + order_by: Iterable[str] | None = None, ): self.order_by = order_by or ['pid'] @abstractmethod - def get_uri( - self - ) -> str: + def get_uri(self) -> str: raise NotImplementedError @abstractmethod def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): raise NotImplementedError @@ -139,37 +137,37 @@ class StorageBackend(metaclass=ABCMeta): @abstractmethod def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: raise NotImplementedError @abstractmethod def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: raise NotImplementedError @abstractmethod def get_records_of_classes( - self, - class_names: Iterable[str], - pattern: str | None = None, + self, + class_names: Iterable[str], + pattern: str | None = None, ) -> BackendResultList: raise NotImplementedError @abstractmethod def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> BackendResultList: raise NotImplementedError def create_sort_key( - json_object: dict[str, Any], - order_by: Iterable[str], + json_object: dict[str, Any], + order_by: Iterable[str], ) -> str: return '-'.join( str(json_object.get(key)) if json_object.get(key) is not None else chr(0x10FFFF) diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index a420fa3..67b94a0 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -45,12 +45,12 @@ class RecordDirResultList(BackendResultList): """ def generate_result( - self, - _: int, - iri: str, - class_name: str, - sort_key: str, - path: Path, + self, + _: int, + iri: str, + class_name: str, + sort_key: str, + path: Path, ) -> RecordInfo: """ Generate a JSON representation of the record at index `index`. @@ -76,11 +76,11 @@ class _RecordDirStore(StorageBackend): """Store records in a directory structure""" def __init__( - self, - root: Path, - pid_mapping_function: Callable, - suffix: str, - order_by: Iterable[str] | None = None, + self, + root: Path, + pid_mapping_function: Callable, + suffix: str, + order_by: Iterable[str] | None = None, ): super().__init__(order_by=order_by) if not root.is_absolute(): @@ -91,28 +91,26 @@ class _RecordDirStore(StorageBackend): self.suffix = suffix self.index = RecordDirIndex(root, suffix) - def get_uri( - self - ) -> str: + def get_uri(self) -> str: return f'file://{self.root!s}' def build_index( - self, - schema: str, + self, + schema: str, ): self.index.rebuild_index(schema, self.order_by) def build_index_if_needed( - self, - schema: str, + self, + schema: str, ): self.index.rebuild_if_needed(schema, self.order_by) def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): pid = json_object['pid'] @@ -148,8 +146,8 @@ class _RecordDirStore(StorageBackend): self.index.add_iri_info(iri, class_name, str(storage_path), sort_string) def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: index_entry = self.index.get_info_for_iri(iri) if index_entry is None: @@ -165,9 +163,9 @@ class _RecordDirStore(StorageBackend): ) def get_records_of_classes( - self, - class_names: list[str], - pattern: str | None = None, + self, + class_names: list[str], + pattern: str | None = None, ) -> RecordDirResultList: return RecordDirResultList().add_info( sorted( @@ -186,8 +184,8 @@ class _RecordDirStore(StorageBackend): ) def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> RecordDirResultList: return RecordDirResultList().add_info( sorted( @@ -205,8 +203,8 @@ class _RecordDirStore(StorageBackend): ) def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: index_entry = self.index.get_info_for_iri(iri) if index_entry is None: @@ -226,10 +224,10 @@ _existing_stores = {} def RecordDirStore( # noqa: N802 - root: Path, - pid_mapping_function: Callable, - suffix: str, - order_by: Iterable[str] | None = None, + root: Path, + pid_mapping_function: Callable, + suffix: str, + order_by: Iterable[str] | None = None, ) -> _RecordDirStore: """Get a record directory store for the given root directory.""" existing_store = _existing_stores.get(root) diff --git a/dump_things_service/backends/record_dir_index.py b/dump_things_service/backends/record_dir_index.py index d872dd6..510d49d 100644 --- a/dump_things_service/backends/record_dir_index.py +++ b/dump_things_service/backends/record_dir_index.py @@ -65,11 +65,11 @@ class IndexEntry(Base): class RecordDirIndex: def __init__( - self, - store_dir: Path, - suffix: str, - *, - echo: bool = False, + self, + store_dir: Path, + suffix: str, + *, + echo: bool = False, ): if not store_dir.is_absolute(): msg = f'Not an absolute path: {store_dir}' @@ -91,11 +91,11 @@ class RecordDirIndex: Base.metadata.create_all(self.engine) def add_iri_info( - self, - iri: str, - class_name: str, - path: str, - sort_key: str, + self, + iri: str, + class_name: str, + path: str, + sort_key: str, ): with Session(self.engine) as session, session.begin(): self.add_iri_info_with_session( @@ -107,12 +107,12 @@ class RecordDirIndex: ) def add_iri_info_with_session( - self, - session: Session, - iri: str, - class_name: str, - path: str, - sort_key: str, + self, + session: Session, + iri: str, + class_name: str, + path: str, + sort_key: str, ): existing_record = session.query(IndexEntry).filter_by(iri=iri).first() if existing_record: @@ -131,8 +131,8 @@ class RecordDirIndex: ) def get_info_for_iri( - self, - iri: str, + self, + iri: str, ) -> tuple | None: with Session(self.engine) as session, session.begin(): statement = select(IndexEntry).filter_by(iri=iri) @@ -142,8 +142,8 @@ class RecordDirIndex: return None def get_info_for_class( - self, - class_name: str, + self, + class_name: str, ) -> Generator[IndexEntry]: with Session(self.engine) as session, session.begin(): statement = select(IndexEntry).filter_by(class_name=class_name) @@ -152,7 +152,7 @@ class RecordDirIndex: yield row[0] def get_info_for_all_classes( - self, + self, ) -> Generator[IndexEntry]: statement = select(IndexEntry) with Session(self.engine) as session, session.begin(): @@ -161,8 +161,8 @@ class RecordDirIndex: yield row[0] def remove_iri_info( - self, - iri: str, + self, + iri: str, ) -> bool: statement = delete(IndexEntry).where(IndexEntry.iri == iri) with Session(self.engine) as session, session.begin(): @@ -170,9 +170,9 @@ class RecordDirIndex: return result.rowcount == 1 def rebuild_index( - self, - schema: str, - order_by: Iterable[str] | None = None, + self, + schema: str, + order_by: Iterable[str] | None = None, ): """Rebuild the index from the records in the directory.""" lgr.info('Building IRI index for records in %s', self.store_dir) @@ -223,17 +223,17 @@ class RecordDirIndex: self.needs_rebuild = False def rebuild_if_needed( - self, - schema: str, - order_by: Iterable[str] | None = None, + self, + schema: str, + order_by: Iterable[str] | None = None, ): if self.needs_rebuild: self.rebuild_index(schema=schema, order_by=order_by) self.needs_rebuild = False def _get_class_name( - self, - path: Path, + self, + path: Path, ) -> str: """Get the class name from the path.""" rel_path = path.absolute().relative_to(self.store_dir) diff --git a/dump_things_service/backends/schema_type_layer.py b/dump_things_service/backends/schema_type_layer.py index 9dd4d18..6ce3594 100644 --- a/dump_things_service/backends/schema_type_layer.py +++ b/dump_things_service/backends/schema_type_layer.py @@ -41,9 +41,9 @@ __all__ = [ class SchemaTypeLayerResultList(BackendResultList): def __init__( - self, - origin_list: BackendResultList, - schema_model: ModuleType, + self, + origin_list: BackendResultList, + schema_model: ModuleType, ): super().__init__() self.schema_model = schema_model @@ -51,12 +51,12 @@ class SchemaTypeLayerResultList(BackendResultList): self.list_info = self.origin_list.list_info def generate_result( - self, - index: int, - iri: str, - class_name: str, - sort_key: str, - private: Any, + self, + index: int, + iri: str, + class_name: str, + sort_key: str, + private: Any, ) -> RecordInfo: origin_element = self.origin_list.generate_result( index, iri, class_name, sort_key, private @@ -73,24 +73,22 @@ class _SchemaTypeLayer(StorageBackend): """Proxy backend that removes `schema_type` from stored records""" def __init__( - self, - backend: StorageBackend, - schema: str, + self, + backend: StorageBackend, + schema: str, ): super().__init__() self.backend = backend self.schema_model = get_schema_model_for_schema(schema) - def get_uri( - self - ) -> str: + def get_uri(self) -> str: return self.backend.get_uri() def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): # Remove the top level `schema_type` from the JSON object because we # don't want to store it in the files. We add `schema_type` after @@ -104,14 +102,14 @@ class _SchemaTypeLayer(StorageBackend): ) def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: return self.backend.remove_record(iri=iri) def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: origin_result = self.backend.get_record_by_iri(iri) if origin_result and 'schema_type' not in origin_result.json_object: @@ -122,9 +120,9 @@ class _SchemaTypeLayer(StorageBackend): return origin_result def get_records_of_classes( - self, - class_names: list[str], - pattern: str | None = None, + self, + class_names: list[str], + pattern: str | None = None, ) -> BackendResultList: return SchemaTypeLayerResultList( origin_list=self.backend.get_records_of_classes( @@ -135,8 +133,8 @@ class _SchemaTypeLayer(StorageBackend): ) def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> BackendResultList: return SchemaTypeLayerResultList( origin_list=self.backend.get_all_records(pattern), @@ -149,8 +147,8 @@ class _SchemaTypeLayer(StorageBackend): def _get_schema_type( - class_name: str, - schema_module: ModuleType, + class_name: str, + schema_module: ModuleType, ) -> str: return getattr(schema_module, class_name).class_class_curie @@ -160,8 +158,8 @@ _existing_layers = {} def SchemaTypeLayer( # noqa: N802 - backend: StorageBackend, - schema: str, + backend: StorageBackend, + schema: str, ) -> _SchemaTypeLayer: existing_layer, _ = _existing_layers.get(id(backend), (None, None)) if not existing_layer: diff --git a/dump_things_service/backends/sqlite.py b/dump_things_service/backends/sqlite.py index b04fd06..38eab9e 100644 --- a/dump_things_service/backends/sqlite.py +++ b/dump_things_service/backends/sqlite.py @@ -88,19 +88,19 @@ class Thing(Base): class SQLResultList(BackendResultList): def __init__( - self, - engine: Any, + self, + engine: Any, ): super().__init__() self.engine = engine def generate_result( - self, - _: int, - iri: str, - class_name: str, - sort_key: str, - db_id: int, + self, + _: int, + iri: str, + class_name: str, + sort_key: str, + db_id: int, ) -> RecordInfo: """ Generate a JSON representation of the record at index `index`. @@ -124,11 +124,11 @@ class SQLResultList(BackendResultList): class _SQLiteBackend(StorageBackend): def __init__( - self, - db_path: Path, - *, - order_by: Iterable[str] | None = None, - echo: bool = False, + self, + db_path: Path, + *, + order_by: Iterable[str] | None = None, + echo: bool = False, ) -> None: assert db_path.is_absolute(), f'db_path not absolute {db_path}' if db_path.exists(): @@ -139,9 +139,7 @@ class _SQLiteBackend(StorageBackend): self.engine = create_engine('sqlite:///' + str(db_path), echo=echo) Base.metadata.create_all(self.engine) - def get_uri( - self - ) -> str: + def get_uri(self) -> str: return f'sqlite://{self.db_path}' def perform_file_name_conversion(self): @@ -152,7 +150,9 @@ class _SQLiteBackend(StorageBackend): logger.info('converting old style name %s', str(old_path)) # Create a backup copy - old_backup_path = (self.db_path.parent / (old_record_file_name + '.backup')).absolute() + old_backup_path = ( + self.db_path.parent / (old_record_file_name + '.backup') + ).absolute() logger.info('copying %s to %s', old_path, old_backup_path) shutil.copyfile(str(old_path), str(old_backup_path)) @@ -161,10 +161,10 @@ class _SQLiteBackend(StorageBackend): shutil.move(str(old_path), str(self.db_path)) def add_record( - self, - iri: str, - class_name: str, - json_object: dict, + self, + iri: str, + class_name: str, + json_object: dict, ): with Session(self.engine) as session, session.begin(): self._add_record_with_session( @@ -175,8 +175,8 @@ class _SQLiteBackend(StorageBackend): ) def add_records_bulk( - self, - record_infos: Iterable[RecordInfo], + self, + record_infos: Iterable[RecordInfo], ): with Session(self.engine) as session, session.begin(): for record_info in record_infos: @@ -188,8 +188,8 @@ class _SQLiteBackend(StorageBackend): ) def remove_record( - self, - iri: str, + self, + iri: str, ) -> bool: statement = delete(Thing).where(Thing.iri == iri) with Session(self.engine) as session, session.begin(): @@ -197,11 +197,11 @@ class _SQLiteBackend(StorageBackend): return result.rowcount == 1 def _add_record_with_session( - self, - session: Session, - iri: str, - class_name: str, - json_object: dict, + self, + session: Session, + iri: str, + class_name: str, + json_object: dict, ): sort_key = create_sort_key(json_object, self.order_by) existing_record = session.query(Thing).filter_by(iri=iri).first() @@ -220,8 +220,8 @@ class _SQLiteBackend(StorageBackend): ) def get_record_by_iri( - self, - iri: str, + self, + iri: str, ) -> RecordInfo | None: with Session(self.engine) as session, session.begin(): statement = select(Thing).filter_by(iri=iri) @@ -236,25 +236,24 @@ class _SQLiteBackend(StorageBackend): return None def get_records_of_classes( - self, - class_names: Iterable[str], - pattern: str | None = None, + self, + class_names: Iterable[str], + pattern: str | None = None, ) -> SQLResultList: - class_list = ', '.join(f"'{cn}'" for cn in class_names) if pattern is None: statement = text( 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'from thing ' - f"where thing.class_name in ({class_list}) " - "ORDER BY thing.sort_key" + f'where thing.class_name in ({class_list}) ' + 'ORDER BY thing.sort_key' ) else: statement = text( 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'from thing, json_tree(thing.object) ' 'where lower(json_tree.value) like lower(:pattern) ' - f"and thing.class_name in ({class_list}) " + f'and thing.class_name in ({class_list}) ' "and json_tree.type = 'text' ORDER BY thing.sort_key" ) @@ -271,14 +270,14 @@ class _SQLiteBackend(StorageBackend): ) def get_all_records( - self, - pattern: str | None = None, + self, + pattern: str | None = None, ) -> SQLResultList: if pattern is None: statement = text( 'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' 'from thing ' - "ORDER BY thing.sort_key" + 'ORDER BY thing.sort_key' ) else: statement = text( @@ -306,7 +305,7 @@ _existing_sqlite_backends = {} def SQLiteBackend( # noqa: N802 - db_path: Path, *, order_by: Iterable[str] | None = None, echo: bool = False + db_path: Path, *, order_by: Iterable[str] | None = None, echo: bool = False ) -> _SQLiteBackend: existing_backend = _existing_sqlite_backends.get(db_path) if not existing_backend: diff --git a/dump_things_service/backends/tests/test_record_dir.py b/dump_things_service/backends/tests/test_record_dir.py index 7b44256..7d4d692 100644 --- a/dump_things_service/backends/tests/test_record_dir.py +++ b/dump_things_service/backends/tests/test_record_dir.py @@ -20,9 +20,7 @@ def test_add_and_delete_record(tmp_path): record_dir_store.build_index(str(schema_path)) record_dir_store.add_record( - iri=iri, - class_name='Object', - json_object={'pid': 'some-pid'} + iri=iri, class_name='Object', json_object={'pid': 'some-pid'} ) record = record_dir_store.get_record_by_iri(iri=iri) diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index 8517f7b..bab7728 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -143,9 +143,9 @@ async def {name}( def create_collection( - instance_state: InstanceState, - configuration: Configuration, - collection_name: str, + instance_state: InstanceState, + configuration: Configuration, + collection_name: str, ): """Create a collection instance as specified by `collection_configuration` @@ -270,10 +270,10 @@ def create_collection( def create_authentication_source( - abstract_configuration: Configuration, - collection_name: str, - authentication_spec: ConfigAuthSpec | ForgejoAuthSpec, - instance_state: InstanceState, + abstract_configuration: Configuration, + collection_name: str, + authentication_spec: ConfigAuthSpec | ForgejoAuthSpec, + instance_state: InstanceState, ): if collection_name not in instance_state.auth_sources: instance_state.auth_sources[collection_name] = [] @@ -301,15 +301,16 @@ def create_authentication_source( def write_record_dir_config( - path: Path, - backend_config: RecordDirBackendConfig, - schema: str, + path: Path, + backend_config: RecordDirBackendConfig, + schema: str, ): assert isinstance(backend_config, RecordDirBackendConfig) record_dir_config_file_path = path / record_dir_config_file_name if not record_dir_config_file_path.exists(): - record_dir_config_file_path.write_text(f"""# RecordDir Config + record_dir_config_file_path.write_text( + f"""# RecordDir Config type: records version: 1 schema: {schema} @@ -320,9 +321,9 @@ idfx: {backend_config.mapping_method} def check_store_compatibility( - store_path: Path, - backend_config: RecordDirBackendConfig | SQLiteBackendConfig, - schema: str, + store_path: Path, + backend_config: RecordDirBackendConfig | SQLiteBackendConfig, + schema: str, ): """Check if an existing store is compatible with the specs in `backend_config` @@ -344,11 +345,10 @@ def check_store_compatibility( def check_record_dir_compatibility( - store_path: Path, - backend_config: RecordDirBackendConfig, - schema: str, + store_path: Path, + backend_config: RecordDirBackendConfig, + schema: str, ): - # Non-existing or empty record_dir-directories are compatible if not store_path.exists(): return @@ -361,7 +361,9 @@ def check_record_dir_compatibility( record_dir_config = get_record_dir_config(store_path) if record_dir_config.schema_location != schema: - raise ConfigCollisionError(f"Existing backend uses a different schema: '{record_dir_config.schema_location}'") + raise ConfigCollisionError( + f"Existing backend uses a different schema: '{record_dir_config.schema_location}'" + ) stored_mapping_method = record_dir_config.idfx.value if stored_mapping_method != backend_config.mapping_method: @@ -371,7 +373,7 @@ def check_record_dir_compatibility( def check_sqlite_compatibility( - store_path: Path, + store_path: Path, ): sqlite_db_path = Path(store_path / sqlite_db_filename) if not sqlite_db_path.exists(): @@ -379,7 +381,7 @@ def check_sqlite_compatibility( def check_git_audit_compatibility( - audit_path: Path, + audit_path: Path, ): """Check if an existing audit path is compatible with a git audit store @@ -408,16 +410,16 @@ def check_git_audit_compatibility( def create_endpoint( - operation_name: str, - operation_path: str, - instance_state: InstanceState, - collection_name: str, - collection_config: CollectionConfig, - template: str, - handler: str, - tag_group: str, - tag_name: str, - app: FastAPI, + operation_name: str, + operation_path: str, + instance_state: InstanceState, + collection_name: str, + collection_config: CollectionConfig, + template: str, + handler: str, + tag_group: str, + tag_name: str, + app: FastAPI, ): logger.info( 'Creating %s-endpoints for collection: "%s"', @@ -428,12 +430,16 @@ def create_endpoint( instance_state.collections[collection_name].tag_info[tag_group] = tag_name # TODO: get schema_info from instance_state!? - model, classes, model_var_name = get_model_for_schema(collection_config.schema_location) + model, classes, model_var_name = get_model_for_schema( + collection_config.schema_location + ) globals()[model_var_name] = model active_classes = instance_state.collections[collection_name].active_classes for class_name in active_classes: - endpoint_name = f'_endpoint_{var_escape(collection_name)}_{operation_name}_{class_name}' + endpoint_name = ( + f'_endpoint_{var_escape(collection_name)}_{operation_name}_{class_name}' + ) endpoint_source = template.format( name=endpoint_name, model_var_name=model_var_name, @@ -451,7 +457,7 @@ def create_endpoint( methods=['POST'], name=f'{operation_name} "{class_name}" object (schema: {model.linkml_meta["id"]})', response_model=None, - tags=[tag_name] + tags=[tag_name], ) logger.info( @@ -462,23 +468,51 @@ def create_endpoint( def create_endpoints_for_collection( - instance_state: InstanceState, - collection_name: str, - collection_config: CollectionConfig, - app: FastAPI, + instance_state: InstanceState, + collection_name: str, + collection_config: CollectionConfig, + app: FastAPI, ): for ( - operation_name, - operation_path, - template, - handler, - tag_group, - tag_name, + operation_name, + operation_path, + template, + handler, + tag_group, + tag_name, ) in ( - ('store', 'record', _endpoint_template, 'store_record', 'write', f'Write records to collection "{collection_name}"'), - ('validate', 'validate/record', _endpoint_template, 'validate_record', 'validate', f'Validate records for collection "{collection_name}"'), - ('curated', 'curated/record', _endpoint_curated_template, 'store_curated_record', 'curated_write', f'Curated area: store records in curated area of collection "{collection_name}"'), - ('incoming', 'incoming/{label}/record', _endpoint_incoming_template, 'store_incoming_record', 'incoming_write', f'Incoming area: store records in incoming area "{{label}}" of collection "{collection_name}"'), + ( + 'store', + 'record', + _endpoint_template, + 'store_record', + 'write', + f'Write records to collection "{collection_name}"', + ), + ( + 'validate', + 'validate/record', + _endpoint_template, + 'validate_record', + 'validate', + f'Validate records for collection "{collection_name}"', + ), + ( + 'curated', + 'curated/record', + _endpoint_curated_template, + 'store_curated_record', + 'curated_write', + f'Curated area: store records in curated area of collection "{collection_name}"', + ), + ( + 'incoming', + 'incoming/{label}/record', + _endpoint_incoming_template, + 'store_incoming_record', + 'incoming_write', + f'Incoming area: store records in incoming area "{{label}}" of collection "{collection_name}"', + ), ): create_endpoint( operation_name=operation_name, @@ -495,17 +529,16 @@ def create_endpoints_for_collection( def delete_endpoints_for_collection( - instance_state: InstanceState, - collection_name: str, + instance_state: InstanceState, + collection_name: str, ): - active_classes = instance_state.collections[collection_name].active_classes for operation_path in ( - 'record', - 'validate/record', - 'curated/record', - 'incoming/{label}/record' + 'record', + 'validate/record', + 'curated/record', + 'incoming/{label}/record', ): delete_endpoint( collection_name=collection_name, @@ -516,10 +549,10 @@ def delete_endpoints_for_collection( def delete_endpoint( - collection_name: str, - active_classes: set[str], - operation_path: str, - app: FastAPI, + collection_name: str, + active_classes: set[str], + operation_path: str, + app: FastAPI, ): from fastapi.routing import _IncludedRouter @@ -539,13 +572,13 @@ def delete_endpoint( def store_record( - collection: str, - data: BaseModel | str, - class_name: str, - model: Any, - input_format: Format, - add_submission_tag: bool, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + data: BaseModel | str, + class_name: str, + model: Any, + input_format: Format, + add_submission_tag: bool, + api_key: str | None = Depends(api_key_header_scheme), ) -> JSONResponse | PlainTextResponse: if input_format == Format.json and isinstance(data, str): raise HTTPException( @@ -591,18 +624,32 @@ def store_record( ) if input_format == Format.ttl: - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Conversion error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Conversion error', + ): json_object = FormatConverter( abstract_config.collections[collection].schema_location, input_format=Format.ttl, output_format=Format.json, ).convert(data, class_name) - with wrap_http_exception(ValidationError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): - record = TypeAdapter(getattr(model, class_name)).validate_python(json_object) + with wrap_http_exception( + ValidationError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): + record = TypeAdapter(getattr(model, class_name)).validate_python( + json_object + ) else: record = data - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(record) with wrap_http_exception(CurieResolutionError): diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index 494f9e7..225c40c 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -69,9 +69,9 @@ class CollectionRequest(CollectionConfig): status_code=HTTP_201_CREATED, ) async def create_collection( - response: Response, - body: CollectionRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: CollectionRequest, + api_key: str = Depends(api_key_header_scheme), ): await create_or_replace_collection(body, api_key, allow_replace=False) response.headers['Location'] = f'/collections/{quote(body.name)}' @@ -84,20 +84,19 @@ async def create_collection( status_code=HTTP_201_CREATED, ) async def replace_collection( - response: Response, - body: CollectionRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: CollectionRequest, + api_key: str = Depends(api_key_header_scheme), ): await create_or_replace_collection(body, api_key, allow_replace=True) response.headers['Location'] = f'/collections/{quote(body.name)}' async def create_or_replace_collection( - body: CollectionRequest, - api_key: str, - allow_replace: bool, + body: CollectionRequest, + api_key: str, + allow_replace: bool, ): - instance_state = get_instance_state() abstract_config = get_config() @@ -165,9 +164,8 @@ async def create_or_replace_collection( name='Get existing collections', ) async def get_collections( - api_key: str = Depends(api_key_header_scheme), + api_key: str = Depends(api_key_header_scheme), ) -> list[CollectionRequest]: - instance_state = get_instance_state() abstract_config = get_config() @@ -177,7 +175,7 @@ async def get_collections( CollectionRequest( **{ 'name': collection_name, - **collection_info.model_dump(mode='json', by_alias=True) + **collection_info.model_dump(mode='json', by_alias=True), } ) for collection_name, collection_info in abstract_config.collections.items() @@ -190,10 +188,9 @@ async def get_collections( name='Get existing collection by name', ) async def get_collection_with_name( - collection_name: str, - api_key: str = Depends(api_key_header_scheme), + collection_name: str, + api_key: str = Depends(api_key_header_scheme), ) -> CollectionConfig: - instance_state = get_instance_state() abstract_config = get_config() @@ -214,10 +211,9 @@ async def get_collection_with_name( name='Delete collection with name', ) async def delete_collection( - collection_name: str, - api_key: str = Depends(api_key_header_scheme), + collection_name: str, + api_key: str = Depends(api_key_header_scheme), ): - instance_state = get_instance_state() abstract_config = get_config() @@ -245,14 +241,16 @@ async def delete_collection( def ensure_unique_directory( - abstract_config: Configuration, - instance_state: InstanceState, - existing_dir: PurePosixPath, + abstract_config: Configuration, + instance_state: InstanceState, + existing_dir: PurePosixPath, ): abs_existing_dir = (instance_state.store_path / Path(existing_dir)).absolute() for collection_name, collection_config in abstract_config.collections.items(): for collection_dir in collection_config.curated, collection_config.incoming: - abs_collection_dir = (instance_state.store_path / Path(collection_dir)).absolute() + abs_collection_dir = ( + instance_state.store_path / Path(collection_dir) + ).absolute() if abs_collection_dir == abs_existing_dir: raise HTTPException( status_code=HTTP_409_CONFLICT, @@ -261,8 +259,8 @@ def ensure_unique_directory( def validate_incoming_paths( - abstract_config: Configuration, - collection_request: CollectionRequest, + abstract_config: Configuration, + collection_request: CollectionRequest, ): for token_name, token_info in abstract_config.tokens.items(): token_collection_info = token_info.collections.get(collection_request.name) @@ -273,7 +271,7 @@ def validate_incoming_paths( detail = ( f"Cannot add collection '{collection_request.name}' without " f"`incoming` path, because at least token '{token_name}' " - f" has write access to the collection" + f' has write access to the collection' ) raise HTTPException( status_code=HTTP_406_NOT_ACCEPTABLE, diff --git a/dump_things_service/commands/check_pids.py b/dump_things_service/commands/check_pids.py index 620e647..46f4c17 100644 --- a/dump_things_service/commands/check_pids.py +++ b/dump_things_service/commands/check_pids.py @@ -25,7 +25,7 @@ from dump_things_service.utils import ( parser = ArgumentParser( prog='Check pids for resolvability', description='This command checks for pids that are in CURIE format and ' - 'cannot be resolved.', + 'cannot be resolved.', ) parser.add_argument( 'store', @@ -43,9 +43,7 @@ def show_backend(model_store: _ModelStore): print(f'Checking: {backend.root}', file=sys.stderr) -def check_pids_in_stores( - stores: Iterable[_ModelStore] -) -> int: +def check_pids_in_stores(stores: Iterable[_ModelStore]) -> int: result = 0 for store in stores: print('checking', store.get_uri(), file=sys.stderr) @@ -61,7 +59,7 @@ def check_pids_in_stores( def check_pids( - store_path: Path, + store_path: Path, ): abstract_config = read_config(store_path) instance_state = create_instance_state( @@ -94,7 +92,7 @@ def check_pids( abstract_config, instance_state, collection, - instance_state.store_path / collection_info.incoming / label + instance_state.store_path / collection_info.incoming / label, ) for label in all_labels ] diff --git a/dump_things_service/commands/create_merged_schema.py b/dump_things_service/commands/create_merged_schema.py index d094765..9ace375 100644 --- a/dump_things_service/commands/create_merged_schema.py +++ b/dump_things_service/commands/create_merged_schema.py @@ -10,16 +10,11 @@ from dump_things_service.patches import enabled # noqa: F401 -- patches LinkML parser = ArgumentParser( prog='Create a static schema with all imported schemas integrated', ) -parser.add_argument( - 'schema', - help='File containing a schema definition' -) +parser.add_argument('schema', help='File containing a schema definition') def update_uris_for_elements( - all_elements: dict, - attribute_name: str, - prefix_index: dict + all_elements: dict, attribute_name: str, prefix_index: dict ): for name, info in all_elements.items(): uri = getattr(info, attribute_name) @@ -32,7 +27,7 @@ def update_uris_for_elements( def update_uris(schema_view: SchemaView): - """ Update element-defining URIs to the original element source + """Update element-defining URIs to the original element source Element-defining URIs (e.g., slot_uri, class_uri) are by default set to the schema in which the element is defined. In this case, that would be the @@ -68,7 +63,7 @@ def main(): Dumper=yaml.SafeDumper, allow_unicode=True, sort_keys=False, - ) + ) print(text) return 0 diff --git a/dump_things_service/commands/download_config.py b/dump_things_service/commands/download_config.py index 2e5ff23..698d6b2 100644 --- a/dump_things_service/commands/download_config.py +++ b/dump_things_service/commands/download_config.py @@ -11,33 +11,35 @@ import yaml parser = ArgumentParser( prog='Download a complete configuration of a running service', description='Read a configuration from dump-things endpoints and create a ' - 'configuration-file that can be possibly modified and uploaded ' - 'to a running service by dump-things-upload-config.' - ' ' - 'An admin token has to be provided in the environment variable ' - '`DTS_ADMIN_TOKEN`.', + 'configuration-file that can be possibly modified and uploaded ' + 'to a running service by dump-things-upload-config.' + ' ' + 'An admin token has to be provided in the environment variable ' + '`DTS_ADMIN_TOKEN`.', ) parser.add_argument( 'server_api', help='The base URL of the server API.', ) parser.add_argument( - '--entities', '-e', + '--entities', + '-e', action='append', choices=['admin_tokens', 'collections', 'tokens'], help='Specify for which entities the configuration should be downloaded. ' - ' Possible values are `admin_tokens`, `collections`, or `tokens` ' - '(repeat to download configuration for more than one entity). If this ' - 'option is not provided, configurations for all entities will be ' - 'downloaded.' + ' Possible values are `admin_tokens`, `collections`, or `tokens` ' + '(repeat to download configuration for more than one entity). If this ' + 'option is not provided, configurations for all entities will be ' + 'downloaded.', ) parser.add_argument( - '--format', '-f', + '--format', + '-f', nargs='?', default='yaml', choices=['json', 'yaml'], help='Specify the format of the output. Possible values are `json` ' - 'and `yaml` (the default is `yaml`).' + 'and `yaml` (the default is `yaml`).', ) @@ -85,11 +87,10 @@ def main(): def get_configuration( - api_url: str, - admin_token: str, - entities: list[str], + api_url: str, + admin_token: str, + entities: list[str], ) -> dict: - result = {} if 'collections' in entities: @@ -109,12 +110,13 @@ def get_configuration( def list_to_dict_on_key( - elements: list[dict], - extract_key: str, + elements: list[dict], + extract_key: str, ) -> dict: return { element[extract_key]: { - element_key: value for element_key, value in element.items() + element_key: value + for element_key, value in element.items() if element_key != extract_key } for element in elements @@ -122,8 +124,8 @@ def list_to_dict_on_key( def get_tokens( - api_url: str, - admin_token: str, + api_url: str, + admin_token: str, ) -> dict: token_list = _get_data( url=api_url + '/tokens', @@ -134,8 +136,8 @@ def get_tokens( def get_collections( - api_url: str, - admin_token: str, + api_url: str, + admin_token: str, ) -> dict: collection_list = _get_data( url=api_url + '/collections', @@ -146,8 +148,8 @@ def get_collections( def get_admin_tokens( - api_url: str, - admin_token: str, + api_url: str, + admin_token: str, ) -> dict: admin_token_list = _get_data( url=api_url + '/admin_tokens', @@ -163,9 +165,9 @@ def get_admin_tokens( def _get_data( - url: str, - token: str, - content_class: str, + url: str, + token: str, + content_class: str, ) -> list: result = requests.get(url, headers={'x-dumpthings-token': token}) if result.status_code >= 300: diff --git a/dump_things_service/commands/gitaudit_rebuild_index.py b/dump_things_service/commands/gitaudit_rebuild_index.py index 5c885c8..30e8489 100644 --- a/dump_things_service/commands/gitaudit_rebuild_index.py +++ b/dump_things_service/commands/gitaudit_rebuild_index.py @@ -8,11 +8,10 @@ from dump_things_service.audit.gitaudit import GitAuditBackend parser = ArgumentParser( prog='Rebuild the index of a `gitaudit`-database', - description='This command rebuilds the index of a `gitaudit`-database.' + description='This command rebuilds the index of a `gitaudit`-database.', ) parser.add_argument( - 'audit_store', - help='The directory in which the `gitaudit`-database is located.' + 'audit_store', help='The directory in which the `gitaudit`-database is located.' ) diff --git a/dump_things_service/commands/gitaudit_report.py b/dump_things_service/commands/gitaudit_report.py index cb92656..a117e36 100644 --- a/dump_things_service/commands/gitaudit_report.py +++ b/dump_things_service/commands/gitaudit_report.py @@ -11,8 +11,8 @@ from dump_things_service.audit.gitaudit import GitAuditBackend parser = ArgumentParser( prog='Report audit information for a PID', description='Report the audit information that was stored for a specific ' - 'PID. For every change to a record the tool will report: ' - 'time stamp, user ID, diff, and the resulting record.', + 'PID. For every change to a record the tool will report: ' + 'time stamp, user ID, diff, and the resulting record.', ) parser.add_argument( 'audit_store', @@ -21,8 +21,8 @@ parser.add_argument( parser.add_argument( 'pid', help='Regex pattern that identifies PIDs of the record for which audit ' - 'information should be reported ' - '(to see all audit log entries, specify ".*").', + 'information should be reported ' + '(to see all audit log entries, specify ".*").', ) diff --git a/dump_things_service/commands/hash_token.py b/dump_things_service/commands/hash_token.py index 3d7312b..b84c912 100644 --- a/dump_things_service/commands/hash_token.py +++ b/dump_things_service/commands/hash_token.py @@ -8,8 +8,8 @@ from dump_things_service.abstract_config import hash_token_representation parser = ArgumentParser( prog='Hash a plain text token to create a hashed token in a dump-things server', description='Hash a token and print the calculated hash value. The hash value ' - 'can be used to create a hashed token via the `/tokens`-endpoint ' - 'of a dump-things-server.', + 'can be used to create a hashed token via the `/tokens`-endpoint ' + 'of a dump-things-server.', ) parser.add_argument( 'token', @@ -29,5 +29,6 @@ def main(): print(hash_token_representation(token)) return 0 + if __name__ == '__main__': sys.exit(main()) diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index 76fb2ae..c215fe1 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -15,48 +15,49 @@ from dump_things_service.instance_state import get_record_dir_config parser = ArgumentParser( prog='Establish a configuration in a running service', description='Read a configuration from a dump-things configuration-file ' - 'and instantiate its elements on a running server. Objects that ' - 'already exist on the server are left unchanged. ' - ' ' - 'An admin token has to be provided in the environment variable ' - '`DTS_ADMIN_TOKEN`.', + 'and instantiate its elements on a running server. Objects that ' + 'already exist on the server are left unchanged. ' + ' ' + 'An admin token has to be provided in the environment variable ' + '`DTS_ADMIN_TOKEN`.', ) parser.add_argument( 'config_file', help='The path to the config file', ) parser.add_argument( - '--format', '-f', + '--format', + '-f', nargs='?', choices=['json', 'yaml'], help='Specify the format of the input file. Possible values are `json` ' - 'and `yaml`. If this option is given, the ' - 'suffix of the configuration file is ignored.' + 'and `yaml`. If this option is given, the ' + 'suffix of the configuration file is ignored.', ) parser.add_argument( '--send-to', help='The base URL of the server API. If this option is provided, the ' - 'configuration will be sent to the server API, otherwise it will just ' - 'be written to stdout.', + 'configuration will be sent to the server API, otherwise it will just ' + 'be written to stdout.', ) parser.add_argument( '--old-format', action='store_true', help='If provided, assume that the configuration is in version 1 format ' - 'and convert it to the new format internally (in version 1: tokens ' - 'had no `hashed`-attribute and no `representation`-attribute, the token ' - 'representation was the key of the token configuration, ' - 'collections had no `schema`-attribute, and `sqlite`-backends had ' - 'a `schema`-attribute).', + 'and convert it to the new format internally (in version 1: tokens ' + 'had no `hashed`-attribute and no `representation`-attribute, the token ' + 'representation was the key of the token configuration, ' + 'collections had no `schema`-attribute, and `sqlite`-backends had ' + 'a `schema`-attribute).', ) parser.add_argument( '--store', default=None, help='If --old-format is provided, this option can be used to specify a ' - 'store directory. The store directory will be used to load `RecordDir` ' - 'configurations, if a collection defines are `RecordDir`-backend. ' - '(This option has no effect if no collection in the old configuration ' - 'uses a `RecordDir`-backend.)', + 'store directory. The store directory will be used to load `RecordDir` ' + 'configurations, if a collection defines are `RecordDir`-backend. ' + '(This option has no effect if no collection in the old configuration ' + 'uses a `RecordDir`-backend.)', ) @@ -92,7 +93,9 @@ def main(): flush=True, ) - assert configuration['type'] == 'collections', '`type: collections` missing in config-file' + assert configuration['type'] == 'collections', ( + '`type: collections` missing in config-file' + ) assert configuration['version'] == 2, '`version: 2` missing in config-file' if arguments.send_to: @@ -131,10 +134,9 @@ def main(): def convert_config_1_to_config_2( - old_configuration: dict, - store_path: str | Path, + old_configuration: dict, + store_path: str | Path, ) -> dict: - old_version = old_configuration.get('version') if old_version != 1: msg = f'`Unknown old configuration format: {old_version}' @@ -150,9 +152,11 @@ def convert_config_1_to_config_2( f'token_{next(counter)}': { **old_token_config.copy(), 'representation': token_representation, - 'hashed': False + 'hashed': False, } - for token_representation, old_token_config in old_configuration['tokens'].items() + for token_representation, old_token_config in old_configuration[ + 'tokens' + ].items() } old_to_new_token_mapping = { @@ -170,14 +174,18 @@ def convert_config_1_to_config_2( if store_path is None: msg = '--store has to be provided to convert collection with record_dir-backends' raise ValueError(msg) - record_dir_config = get_record_dir_config(store_path / collection_config['curated']) + record_dir_config = get_record_dir_config( + store_path / collection_config['curated'] + ) collection_config['schema'] = record_dir_config.schema_location backend = { 'type': 'record_dir+stl' if not backend else backend['type'], - 'mapping_method': record_dir_config.idfx.value + 'mapping_method': record_dir_config.idfx.value, } collection_config['backend'] = backend - collection_config['default_token'] = old_to_new_token_mapping[collection_config['default_token']] + collection_config['default_token'] = old_to_new_token_mapping[ + collection_config['default_token'] + ] new_configuration = { 'type': 'collections', @@ -190,9 +198,9 @@ def convert_config_1_to_config_2( def establish_configuration( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): create_collections(configuration, api_url, admin_token) create_tokens(configuration, api_url, admin_token) @@ -200,9 +208,9 @@ def establish_configuration( def create_tokens( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): for token_name, token_config in configuration['tokens'].items(): _post_data( @@ -218,9 +226,9 @@ def create_tokens( def create_collections( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): for collection_name, collection_config in configuration['collections'].items(): _post_data( @@ -236,9 +244,9 @@ def create_collections( def create_admin_tokens( - configuration: dict, - api_url: str, - admin_token: str, + configuration: dict, + api_url: str, + admin_token: str, ): for admin_token_name, admin_token_config in configuration['admin_tokens'].items(): _post_data( @@ -254,13 +262,17 @@ def create_admin_tokens( def _post_data( - url: str, - data: dict, - token: str, - content_class: str, - content_name: str, + url: str, + data: dict, + token: str, + content_class: str, + content_name: str, ): - result = requests.put(url, headers={'x-dumpthings-token': token}, json=data,) + result = requests.put( + url, + headers={'x-dumpthings-token': token}, + json=data, + ) if result.status_code >= 300: msg = f'Error uploading {content_class}: {content_name}: {result.text}' raise RuntimeError(msg) diff --git a/dump_things_service/converter.py b/dump_things_service/converter.py index 7b8fbb6..c92fbb5 100644 --- a/dump_things_service/converter.py +++ b/dump_things_service/converter.py @@ -46,10 +46,7 @@ class TypeValidator: self.type_name = type_name self.matcher = None if pattern is None else re.compile(pattern) - def validate( - self, - value: str - ) -> str: + def validate(self, value: str) -> str: if self.matcher: match = self.matcher.match(value) if not match: @@ -237,10 +234,7 @@ def _convert_format( ) except Exception as e: # BLE001 if load_only: - msg = ( - f'Validation error for instance of {target_class}: {e}, ' - f'data:\n{data}' - ) + msg = f'Validation error for instance of {target_class}: {e}, data:\n{data}' else: msg = ( f'Conversion {input_format} -> {output_format}. Error: {e}, ' diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index e685d6b..b1e3237 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -74,7 +74,7 @@ add_pagination(router) @router.get( '/{collection}/curated/records/{class_name}', tags=['Curated area: read records'], - name='Read all records of the given class from the curated area' + name='Read all records of the given class from the curated area', ) async def read_curated_records_of_type( collection: str, @@ -102,7 +102,7 @@ async def read_curated_records_of_type( @router.get( '/{collection}/curated/records/p/{class_name}', tags=['Curated area: read records'], - name='Read all records of the given class from the curated area with pagination' + name='Read all records of the given class from the curated area with pagination', ) async def read_curated_records_of_type_paginated( collection: str, @@ -110,7 +110,6 @@ async def read_curated_records_of_type_paginated( matching: str | None = None, api_key: str | None = Depends(api_key_header_scheme), ) -> Page[dict]: - instance_state = get_instance_state() if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( @@ -131,7 +130,7 @@ async def read_curated_records_of_type_paginated( @router.get( '/{collection}/curated/records/', tags=['Curated area: read records'], - name='Read all records from the curated area' + name='Read all records from the curated area', ) async def read_curated_all_records( collection: str, @@ -151,7 +150,7 @@ async def read_curated_all_records( @router.get( '/{collection}/curated/records/p/', tags=['Curated area: read records'], - name='Read all records from the curated area with pagination' + name='Read all records from the curated area with pagination', ) async def read_curated_all_records_paginated( collection: str, @@ -172,7 +171,7 @@ async def read_curated_all_records_paginated( @router.get( '/{collection}/curated/record', tags=['Curated area: read records'], - name='Read the record with the given pid from the curated area' + name='Read the record with the given pid from the curated area', ) async def read_curated_record_with_pid( collection: str, @@ -190,7 +189,7 @@ async def read_curated_record_with_pid( @router.delete( '/{collection}/curated/record', tags=['Curated area: delete records'], - name='Delete the record with the given pid from the curated area of the given collection' + name='Delete the record with the given pid from the curated area of the given collection', ) async def delete_curated_record_with_pid( collection: str, @@ -212,7 +211,6 @@ async def _read_curated_records( api_key: str | None = None, upper_bound: int | None = 1000, ) -> LazyList | dict | None: - model_store, backend, _ = _get_store_and_backend(collection, api_key) if pid: @@ -230,9 +228,7 @@ async def _read_curated_records( len(result_list), upper_bound, collection, - f'/curated/records/p/{class_name}' - if class_name - else '/curated/records/p/', + f'/curated/records/p/{class_name}' if class_name else '/curated/records/p/', ) return ModifierList( @@ -242,9 +238,9 @@ async def _read_curated_records( async def _delete_curated_record( - collection: str, - pid: str | None, - api_key: str | None = None, + collection: str, + pid: str | None, + api_key: str | None = None, ) -> bool: with wrap_http_exception(Exception): model_store, backend, _ = _get_store_and_backend(collection, api_key) @@ -253,7 +249,7 @@ async def _delete_curated_record( raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"Could not remove record with PID '{pid}' from curated area " - f"of collection '{collection}'.", + f"of collection '{collection}'.", ) return True @@ -262,7 +258,6 @@ def _get_store_and_backend( collection: str, plain_token: str | None, ) -> tuple[_ModelStore, StorageBackend, AuthenticationInfo]: - # A token is required if plain_token is None: raise HTTPException( @@ -294,14 +289,18 @@ def _get_store_and_backend( def store_curated_record( - collection: str, - data: BaseModel, - class_name: str, - author_id: str | None = None, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + data: BaseModel, + class_name: str, + author_id: str | None = None, + api_key: str | None = Depends(api_key_header_scheme), ): instance_state = get_instance_state() - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(data) pid = data.pid diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index 1459222..53a5b46 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -55,7 +55,7 @@ add_pagination(router) @router.get( '/{collection}/incoming/', tags=['Incoming area: read labels'], - name='Get all incoming labels for the given collection' + name='Get all incoming labels for the given collection', ) async def incoming_read_labels( collection: str, @@ -66,14 +66,16 @@ async def incoming_read_labels( instance_state = get_instance_state() configured_labels = get_config_labels(get_config(), collection) - on_disk_labels = get_on_disk_labels(instance_state.store_path, get_config(), collection) + on_disk_labels = get_on_disk_labels( + instance_state.store_path, get_config(), collection + ) return list(configured_labels.union(on_disk_labels)) @router.get( '/{collection}/incoming/{label}/records/{class_name}', tags=['Incoming area: read records'], - name='Read all records of the given class from the given incoming area' + name='Read all records of the given class from the given incoming area', ) async def incoming_read_records_of_type( collection: str, @@ -103,7 +105,7 @@ async def incoming_read_records_of_type( @router.get( '/{collection}/incoming/{label}/records/p/{class_name}', tags=['Incoming area: read records'], - name='Read all records of the given class from the given incoming area with pagination' + name='Read all records of the given class from the given incoming area with pagination', ) async def incoming_read_records_of_type_paginated( collection: str, @@ -112,7 +114,6 @@ async def incoming_read_records_of_type_paginated( matching: str | None = None, api_key: str | None = Depends(api_key_header_scheme), ) -> Page[dict]: - instance_state = get_instance_state() if class_name not in instance_state.collections[collection].active_classes: raise HTTPException( @@ -134,7 +135,7 @@ async def incoming_read_records_of_type_paginated( @router.get( '/{collection}/incoming/{label}/records/', tags=['Incoming area: read records'], - name='Read all records from the given incoming area' + name='Read all records from the given incoming area', ) async def incoming_read_all_records( collection: str, @@ -156,13 +157,13 @@ async def incoming_read_all_records( @router.get( '/{collection}/incoming/{label}/records/p/', tags=['Incoming area: read records'], - name='Read all records from the given incoming area with pagination' + name='Read all records from the given incoming area with pagination', ) async def incoming_read_all_records_paginated( - collection: str, - label: str, - matching: str | None = None, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + label: str, + matching: str | None = None, + api_key: str | None = Depends(api_key_header_scheme), ) -> Page[dict]: record_list = await _incoming_read_records( collection=collection, @@ -179,13 +180,13 @@ async def incoming_read_all_records_paginated( @router.get( '/{collection}/incoming/{label}/record', tags=['Incoming area: read records'], - name='Read the record with the given PID from the given incoming area' + name='Read the record with the given PID from the given incoming area', ) async def incoming_read_record_with_pid( - collection: str, - label: str, - pid: str, - api_key: str = Depends(api_key_header_scheme), + collection: str, + label: str, + pid: str, + api_key: str = Depends(api_key_header_scheme), ): return await _incoming_read_records( collection=collection, @@ -199,7 +200,7 @@ async def incoming_read_record_with_pid( @router.delete( '/{collection}/incoming/{label}/record', tags=['Incoming area: delete records'], - name='Delete the record with the given PID from the given incoming area' + name='Delete the record with the given PID from the given incoming area', ) async def incoming_delete_record_with_pid( collection: str, @@ -216,15 +217,14 @@ async def incoming_delete_record_with_pid( async def _incoming_read_records( - collection: str, - label: str, - class_name: str | None, - pid: str | None, - matching: str | None = None, - api_key: str | None = None, - upper_bound: int = 1000, + collection: str, + label: str, + class_name: str | None, + pid: str | None, + matching: str | None = None, + api_key: str | None = None, + upper_bound: int = 1000, ) -> LazyList | dict | None: - model_store, backend = await _get_store_and_backend(collection, label, api_key) if pid: @@ -244,7 +244,7 @@ async def _incoming_read_records( collection, f'/incoming/{label}/records/p/{class_name}' if class_name - else f'/incoming/{label}/records/p/' + else f'/incoming/{label}/records/p/', ) return ModifierList( @@ -266,7 +266,7 @@ async def _incoming_delete_record( raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"Could not remove record with PID '{pid}' from incoming " - f"area '{label}' of collection '{collection}'.", + f"area '{label}' of collection '{collection}'.", ) return True @@ -276,7 +276,6 @@ async def _get_store_and_backend( label: str, plain_token: str | None, ) -> tuple[_ModelStore, StorageBackend]: - # Authorize api_key await authorize_zones(collection, plain_token) @@ -361,15 +360,18 @@ async def authorize_zones( async def store_incoming_record( - collection: str, - label: str, - data: BaseModel, - class_name: str, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + label: str, + data: BaseModel, + class_name: str, + api_key: str | None = Depends(api_key_header_scheme), ): - instance_state = get_instance_state() - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(data) pid = data.pid diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index b82facc..c41dbc3 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -84,7 +84,9 @@ class InstanceState: maintenance_mode: set = dataclasses.field(default_factory=set) # Created based on abstract configuration - collections: dict[str, InstanceStateCollectionInfo] = dataclasses.field(default_factory=dict) + collections: dict[str, InstanceStateCollectionInfo] = dataclasses.field( + default_factory=dict + ) tokens: dict = dataclasses.field(default_factory=dict) auth_sources: dict[str, list] = dataclasses.field(default_factory=dict) audit_backends: dict[str, list] = dataclasses.field(default_factory=dict) @@ -95,13 +97,13 @@ class InstanceState: order_by: list[str] = dataclasses.field(default_factory=list) -g_instance_state:InstanceState | None = None +g_instance_state: InstanceState | None = None def create_instance_state( - store_path: Path, - bootstrap_token: str, - fastapi_app: FastAPI, + store_path: Path, + bootstrap_token: str, + fastapi_app: FastAPI, ) -> InstanceState: global g_instance_state @@ -126,8 +128,8 @@ def get_instance_state() -> InstanceState: def get_record_dir_config( - path: Path, - file_name: str = record_dir_config_file_name, + path: Path, + file_name: str = record_dir_config_file_name, ) -> RecordDirConfigFileContent: config_path = path / file_name if not config_path.exists(): diff --git a/dump_things_service/lazy_list.py b/dump_things_service/lazy_list.py index 35b9bad..8a86c43 100644 --- a/dump_things_service/lazy_list.py +++ b/dump_things_service/lazy_list.py @@ -176,7 +176,7 @@ class PriorityList(LazyList): """ def __init__( - self, + self, ): super().__init__() self.seen = set() diff --git a/dump_things_service/main.py b/dump_things_service/main.py index edfebfc..63f0f51 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING from dump_things_service.abstract_config import store_config from dump_things_service.commands.upload_config import convert_config_1_to_config_2 from dump_things_service.manifest import manifest_configuration + # Perform the patching before importing any third-party libraries from dump_things_service.patches import enabled # noqa: F401 -- used by generated code @@ -57,8 +58,7 @@ from dump_things_service.converter import ( from dump_things_service.curated import router as curated_router from dump_things_service.exceptions import CurieResolutionError from dump_things_service.incoming import router as incoming_router -from dump_things_service.instance_state import create_instance_state, \ - InstanceState +from dump_things_service.instance_state import create_instance_state, InstanceState from dump_things_service.lazy_list import ( PriorityList, ModifierList, @@ -97,7 +97,7 @@ class ServerCollectionCountedResponse(ServerCollectionResponse): class ServerResponse(BaseModel): version: str - collections: list[ServerCollectionResponse|ServerCollectionCountedResponse] + collections: list[ServerCollectionResponse | ServerCollectionCountedResponse] logging.basicConfig(level=logging.WARNING) @@ -114,19 +114,19 @@ parser.add_argument( type=str, default='', help='The sha256 hash of an initial admin token that will allow to add or ' - 'remove tokens, collections, and additional admin tokens (64 ' - 'characters hex-digit). NOTE: an admin token in plaintext is read ' - 'from the environment variable `DTS_ADMIN_TOKEN` if it is set, and ' - 'if this option is not provided.', + 'remove tokens, collections, and additional admin tokens (64 ' + 'characters hex-digit). NOTE: an admin token in plaintext is read ' + 'from the environment variable `DTS_ADMIN_TOKEN` if it is set, and ' + 'if this option is not provided.', ) parser.add_argument( '-c', '--config', metavar='CONFIG_FILE', help="Read the configuration from 'CONFIG_FILE' if no persisted " - "configuration is found in the data store root directory, and " - "initialize the persistent configuration and the service state with " - "the values in 'CONFIG_FILE'.", + 'configuration is found in the data store root directory, and ' + 'initialize the persistent configuration and the service state with ' + "the values in 'CONFIG_FILE'.", ) parser.add_argument( '--root-path', @@ -141,10 +141,10 @@ parser.add_argument( parser.add_argument( '--ignore-default-config-file', action='store_true', - help="If the persisted configuration is empty, do not try to initialize it " - "from an existing default-config file, i.e., do not read the file " - "`/.dumpthings.yaml`. That means the configuration be empty " - "collections and tokens are added via the API.", + help='If the persisted configuration is empty, do not try to initialize it ' + 'from an existing default-config file, i.e., do not read the file ' + '`/.dumpthings.yaml`. That means the configuration be empty ' + 'collections and tokens are added via the API.', ) parser.add_argument( 'store', @@ -246,8 +246,8 @@ g_configuration = read_config(store_path) def initialize_from_config_file( - instance_state: InstanceState, - config_file: str | Path, + instance_state: InstanceState, + config_file: str | Path, ) -> Configuration: with open(config_file) as f: config_dict = yaml.safe_load(f) @@ -273,9 +273,9 @@ def initialize_from_config_file( # location, i.e., from `/.dumpthings.yaml`, or from the configuration # option, unless `--dont-use-old-config` is specified. if not ( - g_configuration.admin_tokens - or g_configuration.collections - or g_configuration.tokens + g_configuration.admin_tokens + or g_configuration.collections + or g_configuration.tokens ): if arguments.config: config_file = arguments.config @@ -283,6 +283,7 @@ if not ( config_file = None else: from dump_things_service import config_file_name + config_file = g_instance_state.store_path / config_file_name if not config_file.exists(): config_file = None @@ -305,9 +306,9 @@ if not ( # If there are no structures in the configuration, check for a bootstrap token. if not ( - g_configuration.admin_tokens - or g_configuration.collections - or g_configuration.tokens + g_configuration.admin_tokens + or g_configuration.collections + or g_configuration.tokens ): if not g_instance_state.bootstrap_token: print( @@ -335,29 +336,27 @@ async def root() -> RedirectResponse: return RedirectResponse('/docs') -@app.get( - '/server', - tags=['Server management'], - name='get server information' -) +@app.get('/server', tags=['Server management'], name='get server information') async def server() -> ServerResponse: return ServerResponse( - version = __version__, - collections = [ + version=__version__, + collections=[ ServerCollectionResponse( name=collection_name, schema=g_configuration.collections[collection_name].schema_location, - classes=g_instance_state.schema_info[g_configuration.collections[collection_name].schema_location].classes, + classes=g_instance_state.schema_info[ + g_configuration.collections[collection_name].schema_location + ].classes, ) for collection_name in g_configuration.collections - ] + ], ) @app.post( '/maintenance', tags=['Server management'], - name='put a collection in maintenance mode' + name='put a collection in maintenance mode', ) async def maintenance( body: MaintenanceRequest, @@ -379,9 +378,9 @@ async def maintenance( permissions = auth_info.token_permission if not ( - permissions.curated_write - and permissions.curated_read - and permissions.zones_access + permissions.curated_write + and permissions.curated_read + and permissions.zones_access ): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, @@ -443,10 +442,10 @@ async def read_record_with_pid( name='Read all records from the given collection', ) async def read_all_records( - collection: str, - matching: str | None = None, - format: Format = Format.json, # noqa: A002 - api_key: str = Depends(api_key_header_scheme), + collection: str, + matching: str | None = None, + format: Format = Format.json, # noqa: A002 + api_key: str = Depends(api_key_header_scheme), ): return await _read_all_records( collection=collection, @@ -466,10 +465,10 @@ async def read_all_records( name='Read all records from the given collection with pagination', ) async def read_all_records_paginated( - collection: str, - matching: str | None = None, - format: Format = Format.json, # noqa: A002 - api_key: str = Depends(api_key_header_scheme), + collection: str, + matching: str | None = None, + format: Format = Format.json, # noqa: A002 + api_key: str = Depends(api_key_header_scheme), ) -> Page[dict | str]: result_list = await _read_all_records( collection=collection, @@ -530,13 +529,12 @@ async def read_records_of_type_paginated( async def _read_all_records( - collection: str, - matching: str | None = None, - format: Format = Format.json, # noqa: A002 - api_key: str = Depends(api_key_header_scheme), - bound: int | None = None, + collection: str, + matching: str | None = None, + format: Format = Format.json, # noqa: A002 + api_key: str = Depends(api_key_header_scheme), + bound: int | None = None, ) -> LazyList: - def convert_to_http_exception(e: BaseException): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, @@ -619,7 +617,9 @@ async def _read_records_of_type( matching=matching, ) if bound: - check_bounds(len(token_store_list), bound, collection, f'/records/p/{class_name}') + check_bounds( + len(token_store_list), bound, collection, f'/records/p/{class_name}' + ) result_list.add_list(token_store_list) if final_permissions.curated_read: @@ -631,7 +631,12 @@ async def _read_records_of_type( matching=matching, ) if bound: - check_bounds(len(curated_store_list), bound, collection, f'/records/p/{class_name}') + check_bounds( + len(curated_store_list), + bound, + collection, + f'/records/p/{class_name}', + ) result_list.add_list(curated_store_list) # Sort the result list. @@ -679,8 +684,8 @@ async def delete_record( raise HTTPException( status_code=HTTP_404_NOT_FOUND, detail=f"Could not remove record with PID '{pid}' from the " - "token associated incoming area of collection " - f"'{collection}'.", + 'token associated incoming area of collection ' + f"'{collection}'.", ) return True diff --git a/dump_things_service/manifest.py b/dump_things_service/manifest.py index a1acc44..6d4e6de 100644 --- a/dump_things_service/manifest.py +++ b/dump_things_service/manifest.py @@ -57,10 +57,9 @@ openapi_tags_template = [ ] - def manifest_configuration( - configuration: Configuration, - instance_state: InstanceState, + configuration: Configuration, + instance_state: InstanceState, ): """Interpret the configuration and instantiate respective objects @@ -159,23 +158,23 @@ def manifest_configuration( def create_token( - instance_state: InstanceState, - token_name: str, - token_configuration: TokenConfig, + instance_state: InstanceState, + token_name: str, + token_configuration: TokenConfig, ): instance_state.tokens[token_name] = token_configuration def delete_token( - instance_state: InstanceState, - token_name: str, + instance_state: InstanceState, + token_name: str, ): instance_state.tokens.pop(token_name) def delete_collection( - instance_state: InstanceState, - collection_name: str, + instance_state: InstanceState, + collection_name: str, ): instance_state.collections.pop(collection_name) @@ -189,7 +188,6 @@ def create_openapi_tags( instance_state: InstanceState, openapi_tags_template: list[dict | str], ) -> list[dict]: - # Collect tag name lists for all tag groups that we have defined. tag_group_info = { tag_group: sorted( @@ -197,12 +195,12 @@ def create_openapi_tags( {'name': collection_info.tag_info[tag_group]} for collection_info in instance_state.collections.values() ], - key=lambda x: x['name'] + key=lambda x: x['name'], ) for tag_group in tag_groups } result = openapi_tags_template.copy() for tag_group, tag_list in tag_group_info.items(): index = result.index(tag_group) - result[index:index + 1] = tag_list + result[index : index + 1] = tag_list return result diff --git a/dump_things_service/model.py b/dump_things_service/model.py index d6311a5..94f0daf 100644 --- a/dump_things_service/model.py +++ b/dump_things_service/model.py @@ -70,8 +70,8 @@ def get_subclasses( # go through the pydantic module generation. @cache def get_subclasses_2( - collection_name: str, - class_name: str, + collection_name: str, + class_name: str, ) -> list[str]: from dump_things_service.instance_state import get_instance_state @@ -88,8 +88,8 @@ def compile_module_with_increasing_recursion_limit( module = None module_name = ( - urlparse(schema_location).path - .replace('/', '_') + urlparse(schema_location) + .path.replace('/', '_') .replace('-', '_') .replace('.', '_') ) diff --git a/dump_things_service/store/model_store.py b/dump_things_service/store/model_store.py index 787258b..489a3b5 100644 --- a/dump_things_service/store/model_store.py +++ b/dump_things_service/store/model_store.py @@ -28,12 +28,7 @@ submitter_namespace = 'http://purl.obolibrary.org/obo/' class _ModelStore: - def __init__( - self, - schema: str, - backend: StorageBackend, - tags: dict[str, str] - ): + def __init__(self, schema: str, backend: StorageBackend, tags: dict[str, str]): self.schema = schema self.model = get_model_for_schema(self.schema)[0] self.backend = backend @@ -43,11 +38,13 @@ class _ModelStore: return self.backend.get_uri() def store_object( - self, - obj: BaseModel, - submitter: str | None, + self, + obj: BaseModel, + submitter: str | None, ) -> Iterable[tuple[str, dict]]: - if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in (obj.annotations or dict()): + if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in ( + obj.annotations or dict() + ): return [] # Extract inlined records from the object, store individual records @@ -64,15 +61,15 @@ class _ModelStore: ] def pid_to_iri( - self, - pid: str, + self, + pid: str, ): return resolve_curie(self.model, pid) def _store_flat_object( - self, - obj: BaseModel, - submitter: str | None, + self, + obj: BaseModel, + submitter: str | None, ) -> dict: iri = self.pid_to_iri(obj.pid) class_name = obj.__class__.__name__ @@ -94,9 +91,9 @@ class _ModelStore: return json_object def annotate( - self, - json_object: dict, - submitter: str, + self, + json_object: dict, + submitter: str, ) -> None: """Add submitter IRI to the record annotations, use CURIE if possible""" json_object['annotations'] = self.homogenize_annotations(json_object) @@ -113,8 +110,8 @@ class _ModelStore: } def get_curie( - self, - curie_or_iri: str, + self, + curie_or_iri: str, ) -> str: if is_curie(curie_or_iri): return curie_or_iri @@ -131,8 +128,8 @@ class _ModelStore: return curie_or_iri def extract_inlined( - self, - record: BaseModel, + self, + record: BaseModel, ) -> list[BaseModel]: # The trivial case: no relations if not hasattr(record, 'relations') or record.relations is None: @@ -146,7 +143,8 @@ class _ModelStore: # Do not extract 'empty'-Thing records with an # `dlthings:placeholder` annotation. These records are just # placeholders for already extracted records. - if sub_record != self.model.Thing( + if sub_record + != self.model.Thing( pid=sub_record.pid, annotations={ 'dlthings:placeholder': sub_record.pid, @@ -165,21 +163,21 @@ class _ModelStore: pid=sub_record_pid, annotations={ 'dlthings:placeholder': sub_record_pid, - } + }, ) for sub_record_pid in record.relations } return [new_record, *extracted_sub_records] def get_object_by_pid( - self, - pid: str, + self, + pid: str, ) -> tuple[str, dict] | tuple[None, None]: return self.get_object_by_iri(self.pid_to_iri(pid)) def get_object_by_iri( - self, - iri: str, + self, + iri: str, ) -> tuple[str, dict] | tuple[None, None]: record_info = self.backend.get_record_by_iri(iri) if record_info: @@ -187,11 +185,11 @@ class _ModelStore: return None, None def get_objects_of_class( - self, - class_name: str, - matching: str | None, - *, - include_subclasses: bool = True, + self, + class_name: str, + matching: str | None, + *, + include_subclasses: bool = True, ) -> LazyList[_RecordInfo]: """ Get all objects of a specific class. @@ -210,8 +208,8 @@ class _ModelStore: return self.backend.get_records_of_classes(class_names, matching) def get_all_objects( - self, - matching: str | None = None, + self, + matching: str | None = None, ) -> LazyList[_RecordInfo]: """ Get all objects of a specific class. @@ -222,8 +220,8 @@ class _ModelStore: return self.backend.get_all_records(matching) def delete_object( - self, - pid: str, + self, + pid: str, ) -> bool: return self.backend.remove_record(self.pid_to_iri(pid)) @@ -232,9 +230,9 @@ _existing_model_stores = {} def ModelStore( # noqa: N802 - schema: str, - backend: StorageBackend, - tags: dict[str, str], + schema: str, + backend: StorageBackend, + tags: dict[str, str], ) -> _ModelStore: """Create a unique model store for the given schema and backend. diff --git a/dump_things_service/tests/fixtures.py b/dump_things_service/tests/fixtures.py index 127c4cd..e48cc79 100644 --- a/dump_things_service/tests/fixtures.py +++ b/dump_things_service/tests/fixtures.py @@ -44,7 +44,9 @@ incoming = 'incoming' # Path to a local simple test schema test_schema_location = str((Path(__file__).parent / 'testschema.yaml').absolute()) -flat_social_schema_location = 'https://concepts.datalad.org/s/flat-social/unreleased.yaml' +flat_social_schema_location = ( + 'https://concepts.datalad.org/s/flat-social/unreleased.yaml' +) # The test store is created empty and collections are added via the admin @@ -78,38 +80,40 @@ g_default_collections.append( submission_tags=TagSpec( submitter_id_tag='no_default_id_tag', submission_time_tag='no_default_time_tag', - ) + ), ) ) -g_default_collections.extend([ - CollectionRequest( - name='collection_dlflatsocial-1', - schema=flat_social_schema_location, - default_token='test_default_token', - curated=PurePosixPath(f'{curated}/collection_dlflatsocial-1'), - incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-1'), - ), - CollectionRequest( - name='collection_dlflatsocial-2', - schema=flat_social_schema_location, - default_token='test_default_token', - curated=PurePosixPath(f'{curated}/collection_dlflatsocial-2'), - incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-2'), - backend=SQLiteBackendConfig( - type='sqlite', +g_default_collections.extend( + [ + CollectionRequest( + name='collection_dlflatsocial-1', + schema=flat_social_schema_location, + default_token='test_default_token', + curated=PurePosixPath(f'{curated}/collection_dlflatsocial-1'), + incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-1'), ), - use_classes=[ - 'Organization', - 'Person', - 'Project', - ], - ignore_classes=[ - 'Organization', - 'Project', - ], - ), -]) + CollectionRequest( + name='collection_dlflatsocial-2', + schema=flat_social_schema_location, + default_token='test_default_token', + curated=PurePosixPath(f'{curated}/collection_dlflatsocial-2'), + incoming=PurePosixPath(f'{incoming}/collection_dlflatsocial-2'), + backend=SQLiteBackendConfig( + type='sqlite', + ), + use_classes=[ + 'Organization', + 'Person', + 'Project', + ], + ignore_classes=[ + 'Organization', + 'Project', + ], + ), + ] +) g_default_tokens = [ TokenRequest( @@ -238,7 +242,7 @@ g_default_tokens = [ mode=TokenModes.WRITE_COLLECTION, incoming_label='modes', ), - } + }, ), TokenRequest( name='Test 0X000 (READ_SUBMISSIONS)', @@ -357,7 +361,8 @@ def fastapi_app_simple(dump_stores_simple): old_sys_argv = sys.argv sys.argv = [ 'test-runner', - '--admin-token-hash', hash_token_representation(admin_token), + '--admin-token-hash', + hash_token_representation(admin_token), '--ignore-default-config-file', str(tmp_path), ] @@ -432,15 +437,15 @@ def fastapi_client_simple(fastapi_app_simple): def add_records_to_backend( - backend: StorageBackend, - pydantic_module: ModuleType, - record_infos: list[tuple[str, str, str]], + backend: StorageBackend, + pydantic_module: ModuleType, + record_infos: list[tuple[str, str, str]], ): for class_name, record_pid, yaml_stream in record_infos: - json_object = yaml.load(yaml_stream, Loader=yaml.SafeLoader ) + json_object = yaml.load(yaml_stream, Loader=yaml.SafeLoader) assert record_pid == json_object['pid'] backend.add_record( iri=resolve_curie(pydantic_module, json_object['pid']), class_name=class_name, json_object=json_object, - ) + ) diff --git a/dump_things_service/tests/test_auth.py b/dump_things_service/tests/test_auth.py index efba9be..f970d73 100644 --- a/dump_things_service/tests/test_auth.py +++ b/dump_things_service/tests/test_auth.py @@ -15,11 +15,7 @@ user_1 = { '@type': 'user', } -org_1 = { - 'id': 1, - 'name': 'org_1', - '@type': 'org' -} +org_1 = {'id': 1, 'name': 'org_1', '@type': 'org'} repo_1 = { 'id': 3, @@ -46,10 +42,18 @@ team_3 = json.loads(team_template.format(id=3, action='write')) def setup_http_server(http_server) -> None: for instance in ('1', '2'): http_server.expect_request(f'/api/v{instance}/user').respond_with_json(user_1) - http_server.expect_request(f'/api/v{instance}/user/teams').respond_with_json([team_1, team_3]) - http_server.expect_request(f'/api/v{instance}/orgs/org_1').respond_with_json(org_1) - http_server.expect_request(f'/api/v{instance}/orgs/org_1/teams').respond_with_json([team_1, team_2, team_3]) - http_server.expect_request(f'/api/v{instance}/repos/org_1/repo_1/teams').respond_with_json([team_1, team_2, team_3]) + http_server.expect_request(f'/api/v{instance}/user/teams').respond_with_json( + [team_1, team_3] + ) + http_server.expect_request(f'/api/v{instance}/orgs/org_1').respond_with_json( + org_1 + ) + http_server.expect_request( + f'/api/v{instance}/orgs/org_1/teams' + ).respond_with_json([team_1, team_2, team_3]) + http_server.expect_request( + f'/api/v{instance}/repos/org_1/repo_1/teams' + ).respond_with_json([team_1, team_2, team_3]) @pytest.mark.parametrize('repository', ['repo_1', None]) diff --git a/dump_things_service/tests/test_basic.py b/dump_things_service/tests/test_basic.py index 7d3e6ea..7c9c966 100644 --- a/dump_things_service/tests/test_basic.py +++ b/dump_things_service/tests/test_basic.py @@ -1,5 +1,3 @@ - - from .. import ( HTTP_200_OK, HTTP_400_BAD_REQUEST, @@ -390,7 +388,8 @@ def test_server(fastapi_client_simple): 'classes': test_schema_classes, } for i in range(1, 9) - ] + [ + ] + + [ { 'name': f'collection_dlflatsocial-{i}', 'schema': 'https://concepts.datalad.org/s/flat-social/unreleased.yaml', diff --git a/dump_things_service/tests/test_collection_administration.py b/dump_things_service/tests/test_collection_administration.py index 3e2f41c..331efbb 100644 --- a/dump_things_service/tests/test_collection_administration.py +++ b/dump_things_service/tests/test_collection_administration.py @@ -54,7 +54,7 @@ new_token_request = TokenRequest( }, ) -new_admin_token_name='New_Admin_Token' +new_admin_token_name = 'New_Admin_Token' plain_new_admin_token = 'admin-XXX' new_admin_token_request = AdminTokenRequest( name=new_admin_token_name, @@ -63,8 +63,8 @@ new_admin_token_request = AdminTokenRequest( def _name_in_openapi_paths( - test_client: TestClient, - name: str, + test_client: TestClient, + name: str, ) -> bool: response = test_client.get('/openapi.json') open_api = response.json() @@ -99,7 +99,9 @@ def test_collection_adding(fastapi_client_simple): headers={'x-dumpthings-token': admin_token}, ) assert response.status_code == HTTP_200_OK - new_collection_config = new_collection_request.model_dump(mode='json', by_alias=True) + new_collection_config = new_collection_request.model_dump( + mode='json', by_alias=True + ) del new_collection_config['name'] assert response.json() == new_collection_config @@ -122,7 +124,7 @@ def test_collection_adding(fastapi_client_simple): 'user_id': new_token_request.user_id, 'collections': new_token_request.model_dump(mode='json')['collections'], 'hashed': new_token_request.hashed, - 'representation': new_token_request.representation + 'representation': new_token_request.representation, } new_record = { @@ -203,7 +205,7 @@ def test_collection_putting(fastapi_client_simple, tmp_path): path=Path(tmp_path), auto_flush_timeout=2, ) - ] + ], ) # Check that the collection does not yet exist diff --git a/dump_things_service/tests/test_config.py b/dump_things_service/tests/test_config.py index b2c36ef..2cc88d5 100644 --- a/dump_things_service/tests/test_config.py +++ b/dump_things_service/tests/test_config.py @@ -35,10 +35,10 @@ def test_illegal_collection_name_detection(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple for name in ( - 'collections', - 'tokens', - 'admin_tokens', - dump_things_private_collection_name, + 'collections', + 'tokens', + 'admin_tokens', + dump_things_private_collection_name, ): response = test_client.post( '/collections', @@ -51,14 +51,16 @@ def test_illegal_collection_name_detection(fastapi_client_simple): assert response.status_code == HTTP_409_CONFLICT -@pytest.mark.skip(reason='Reuse detection is disabled to support existing old configurations') +@pytest.mark.skip( + reason='Reuse detection is disabled to support existing old configurations' +) def test_collection_dir_reuse_detection(fastapi_client_simple): test_client, _, admin_token = fastapi_client_simple for curated_path, incoming_path in ( - ('curated/collection_1', 'incoming/XXXX'), - ('curated/XXXX', 'incoming/collection_1'), - ('curated/collection_1', 'incoming/collection_2'), + ('curated/collection_1', 'incoming/XXXX'), + ('curated/XXXX', 'incoming/collection_1'), + ('curated/collection_1', 'incoming/collection_2'), ): response = test_client.post( '/collections', @@ -79,11 +81,13 @@ def test_scanner_error_detection(tmp_path_factory): config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object={'pid': dump_things_config_iri} + json_object={'pid': dump_things_config_iri}, ) md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest() - config_file_path = config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + config_file_path = ( + config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + ) config_file_path.write_text('collections: ::: -\n sdsdfsdf: xxx') with pytest.raises(ConfigError): read_config(tmp_path, force_reload=True) @@ -96,11 +100,13 @@ def test_structure_error_detection(tmp_path_factory): config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', - json_object={'pid': dump_things_config_iri} + json_object={'pid': dump_things_config_iri}, ) md5_hexdigest = hashlib.md5(dump_things_config_iri.encode()).hexdigest() - config_file_path = config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + config_file_path = ( + config_backend.root / 'DumpThingsConfig' / f'{md5_hexdigest}.yaml' + ) config_file_path.write_text('type: 1\n') with pytest.raises(ConfigError): read_config(tmp_path, force_reload=True) @@ -134,7 +140,7 @@ def test_missing_incoming_detection(fastapi_client_simple): mode=TokenModes.CURATOR, incoming_label='', ) - } + }, ) # Check that a write token for a collection without incoming path cannot @@ -154,7 +160,9 @@ def test_missing_incoming_detection(fastapi_client_simple): assert response.status_code == HTTP_200_OK # Add a collection with incoming path - collection_request.incoming = PurePosixPath('missing_incoming_detection_test_incoming') + collection_request.incoming = PurePosixPath( + 'missing_incoming_detection_test_incoming' + ) response = test_client.post( '/collections', json=collection_request.model_dump(mode='json', by_alias=True), @@ -172,9 +180,11 @@ def test_missing_incoming_detection(fastapi_client_simple): assert response.status_code == HTTP_406_NOT_ACCEPTABLE # Check that a write token for a collection with an incoming path can be created - token_request.collections['missing_incoming_detection_test'] = TokenCollectionConfig( - mode=TokenModes.CURATOR, - incoming_label='test_incoming_label', + token_request.collections['missing_incoming_detection_test'] = ( + TokenCollectionConfig( + mode=TokenModes.CURATOR, + incoming_label='test_incoming_label', + ) ) response = test_client.post( '/tokens', diff --git a/dump_things_service/tests/test_curated.py b/dump_things_service/tests/test_curated.py index 1c34f04..8ae7ff5 100644 --- a/dump_things_service/tests/test_curated.py +++ b/dump_things_service/tests/test_curated.py @@ -58,6 +58,8 @@ pytest.mark.parametrize( 'pid', ('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'), ) + + def test_read_curated_records_by_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple diff --git a/dump_things_service/tests/test_extract_inline.py b/dump_things_service/tests/test_extract_inline.py index 02c7440..d88be3c 100644 --- a/dump_things_service/tests/test_extract_inline.py +++ b/dump_things_service/tests/test_extract_inline.py @@ -113,7 +113,10 @@ empty_inlined_json_record = cleaned_json(dataclasses.asdict(empty_inlined_object tree = ( - ('dlflatsocial:test_extract_1', ('dlflatsocial:test_extract_1_1', 'dlflatsocial:test_extract_1_2')), + ( + 'dlflatsocial:test_extract_1', + ('dlflatsocial:test_extract_1_1', 'dlflatsocial:test_extract_1_2'), + ), ('dlflatsocial:test_extract_1_1', ('dlflatsocial:test_extract_1_1_1',)), ('dlflatsocial:test_extract_1_2', ()), ('dlflatsocial:test_extract_1_1_1', ()), @@ -181,10 +184,10 @@ def test_inline_extraction_locally(): store = ModelStore( schema=str(schema_path), backend=None, - tags = { + tags={ 'id': 'abc:id', 'time': 'abc:time', - } + }, ) store.model = MockedModule() records = store.extract_inlined(inlined_object) @@ -216,7 +219,7 @@ def test_dont_extract_empty_things_locally(): tags={ 'id': 'https://id', 'time': 'https://time', - } + }, ) store.model = MockedModule() records = store.extract_inlined(empty_inlined_object) @@ -257,7 +260,10 @@ def test_inline_extraction_on_service(fastapi_client_simple): # Check that individual record classes were recognized for class_name, pids in ( - ('Person', ('dlflatsocial:test_extract_1', 'dlflatsocial:test_extract_1_1')), + ( + 'Person', + ('dlflatsocial:test_extract_1', 'dlflatsocial:test_extract_1_1'), + ), ('Agent', ('dlflatsocial:test_extract_1_1_1',)), ('InstantaneousEvent', ('dlflatsocial:test_extract_1_2',)), ): @@ -301,7 +307,10 @@ def test_inline_ttl_processing(fastapi_client_simple): # Check that individual record classes were recognized for class_name, pids in ( - ('Person', ('dlflatsocial:test_ttl_inline_1', 'dlflatsocial:test_ttl_inline_1_1')), + ( + 'Person', + ('dlflatsocial:test_ttl_inline_1', 'dlflatsocial:test_ttl_inline_1_1'), + ), ('Agent', ('dlflatsocial:test_ttl_inline_1_1_1',)), ('InstantaneousEvent', ('dlflatsocial:test_ttl_inline_1_2',)), ): @@ -386,9 +395,9 @@ def test_store_complex_things(fastapi_client_simple): 'http://test.complex.thing/1.1.1': { 'pid': 'http://test.complex.thing/1.1.1', } - } + }, } - } + }, } # Deposit JSON record @@ -402,9 +411,9 @@ def test_store_complex_things(fastapi_client_simple): # Try to read individual extracted elements for pid in ( - 'http://test.complex.thing/1', - 'http://test.complex.thing/1.1', - 'http://test.complex.thing/1.1.1', + 'http://test.complex.thing/1', + 'http://test.complex.thing/1.1', + 'http://test.complex.thing/1.1.1', ): response = test_client.get( f'/collection_1/record?pid={pid}', diff --git a/dump_things_service/tests/test_ifabsent_patch.py b/dump_things_service/tests/test_ifabsent_patch.py index 5146f01..e1f4312 100644 --- a/dump_things_service/tests/test_ifabsent_patch.py +++ b/dump_things_service/tests/test_ifabsent_patch.py @@ -16,10 +16,11 @@ def _original_uri_for(self, s: str) -> str: def test_ifabsent_patch(): - # Patch in the faulty, original code and check for its result if_abs_proc.IfAbsentProcessor._uri_for = _original_uri_for - gen1 = linkml.generators.PydanticGenerator(str(schema_dir / 'schema-ifabsent-error.yaml')) + gen1 = linkml.generators.PydanticGenerator( + str(schema_dir / 'schema-ifabsent-error.yaml') + ) x = gen1.serialize() assert 'default=XSD["04fa4r544"]' in x @@ -27,6 +28,8 @@ def test_ifabsent_patch(): reload(dump_things_service.patches.ifabsent_processing) # Check for proper code generation - gen2 = linkml.generators.PydanticGenerator(str(schema_dir / 'schema-ifabsent-error.yaml')) + gen2 = linkml.generators.PydanticGenerator( + str(schema_dir / 'schema-ifabsent-error.yaml') + ) y = gen2.serialize() assert 'XSD' not in y diff --git a/dump_things_service/tests/test_incoming.py b/dump_things_service/tests/test_incoming.py index 15b27a4..ecc7cad 100644 --- a/dump_things_service/tests/test_incoming.py +++ b/dump_things_service/tests/test_incoming.py @@ -30,6 +30,7 @@ def test_incoming_labels(fastapi_client_simple): zones_filled = False + def fill_zones(test_client): global zones_filled @@ -53,7 +54,7 @@ def fill_zones(test_client): json={ 'pid': f'abc:test_incoming-collection_{collection_id}-{token}', 'given_name': f'collection_{collection_id}-{token}', - } + }, ) assert result.status_code == HTTP_200_OK @@ -63,23 +64,23 @@ def fill_zones(test_client): @pytest.mark.parametrize('paginate', ('', 'p/')) @pytest.mark.parametrize('class_name', ('', 'Person')) def test_read_incoming_records( - fastapi_client_simple, - paginate: str, - class_name: str, + fastapi_client_simple, + paginate: str, + class_name: str, ): test_client, _, _ = fastapi_client_simple fill_zones(test_client) for collection_id, labels in ( - (1, ['modes', 'admin_1', 'in_token_1']), - (2, ['in_token-2', 'admin_2']), - (3, ['admin_3']), - (4, ['admin_4']), - (5, ['admin_common']), - (6, ['admin_common']), - (7, ['admin_common']), - (8, ['modes', 'test_user_8', 'admin_common']), + (1, ['modes', 'admin_1', 'in_token_1']), + (2, ['in_token-2', 'admin_2']), + (3, ['admin_3']), + (4, ['admin_4']), + (5, ['admin_common']), + (6, ['admin_common']), + (7, ['admin_common']), + (8, ['modes', 'test_user_8', 'admin_common']), ): # Check that all incoming zones are reached for label in labels: @@ -87,7 +88,9 @@ def test_read_incoming_records( f'/collection_{collection_id}/incoming/{label}/records/{paginate}{class_name}', headers={'x-dumpthings-token': 'token_curator'}, ) - assert response.status_code == HTTP_200_OK, f'failed on collection: {collection_id}, label: {label}, class: {class_name}' + assert response.status_code == HTTP_200_OK, ( + f'failed on collection: {collection_id}, label: {label}, class: {class_name}' + ) # We don't know the exact number of entries in each zone, because # it depends on the tests that ran before. @@ -108,17 +111,19 @@ def test_read_incoming_records( else: result = json_object matching = [ - json_object - for json_object in result - if json_object['pid'] == pattern + json_object for json_object in result if json_object['pid'] == pattern ] - assert len(matching) == expected_length, f'did not find {expected_length} record: collection_{collection_id}, {label}, {result}' + assert len(matching) == expected_length, ( + f'did not find {expected_length} record: collection_{collection_id}, {label}, {result}' + ) pytest.mark.parametrize( 'pid', ('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'), ) + + def test_read_incoming_records_by_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple diff --git a/dump_things_service/tests/test_roundtrip_flatsocial.py b/dump_things_service/tests/test_roundtrip_flatsocial.py index 83ac558..f1e1797 100644 --- a/dump_things_service/tests/test_roundtrip_flatsocial.py +++ b/dump_things_service/tests/test_roundtrip_flatsocial.py @@ -143,7 +143,9 @@ def test_ttl_json_ttl_dlflatsocial(fastapi_client_simple): }, data=ttl_input_record, ) - assert response.status_code == HTTP_200_OK, 'Response content: ' + response.content.decode() + assert response.status_code == HTTP_200_OK, ( + 'Response content: ' + response.content.decode() + ) # Retrieve JSON records response = test_client.get( @@ -171,8 +173,12 @@ def test_ttl_json_ttl_dlflatsocial(fastapi_client_simple): assert response.status_code == HTTP_200_OK assert ( response.text.strip() - == ttl_output_record_a.replace('dlflatsocial:test_john_ttl', new_json_pid).strip() + == ttl_output_record_a.replace( + 'dlflatsocial:test_john_ttl', new_json_pid + ).strip() ) or ( response.text.strip() - == ttl_output_record_b.replace('dlflatsocial:test_john_ttl', new_json_pid).strip() + == ttl_output_record_b.replace( + 'dlflatsocial:test_john_ttl', new_json_pid + ).strip() ) diff --git a/dump_things_service/tests/test_token_endpoints.py b/dump_things_service/tests/test_token_endpoints.py index 20ece81..0af5019 100644 --- a/dump_things_service/tests/test_token_endpoints.py +++ b/dump_things_service/tests/test_token_endpoints.py @@ -11,11 +11,8 @@ def test_token_creation(fastapi_client_simple): 'user_id': 'u_a', 'representation': '8bb6805ff10bcb1c2ca49dcd4bfef94d', 'collections': { - 'collection_1': { - 'mode': 'WRITE_COLLECTION', - 'incoming_label': 'i_a' - } - } + 'collection_1': {'mode': 'WRITE_COLLECTION', 'incoming_label': 'i_a'} + }, } # Create a token with name 'a' diff --git a/dump_things_service/tests/test_unicode.py b/dump_things_service/tests/test_unicode.py index 5a1deb6..e26e313 100644 --- a/dump_things_service/tests/test_unicode.py +++ b/dump_things_service/tests/test_unicode.py @@ -30,9 +30,9 @@ def test_unicode_iri(fastapi_client_simple): response = test_client.post( '/collection_1/record/Person', headers={'x-dumpthings-token': 'token-1'}, - json = { + json={ 'pid': 'https://en.wikipedia.org/wiki/Universita_degli_Studi_eCampus', - 'given_name': 'Università degli Studi eCampus (Italy)', # codespell:ignore - } + 'given_name': 'Università degli Studi eCampus (Italy)', # codespell:ignore + }, ) assert response.status_code == HTTP_200_OK diff --git a/dump_things_service/tests/test_validate.py b/dump_things_service/tests/test_validate.py index 800b17c..d6c3e10 100644 --- a/dump_things_service/tests/test_validate.py +++ b/dump_things_service/tests/test_validate.py @@ -1,9 +1,11 @@ - from dump_things_service import HTTP_422_UNPROCESSABLE_CONTENT json_records = [ ({'name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT), - ({'given_name': 'Henry', 'pid': 'unknown_prefix:henry'}, HTTP_422_UNPROCESSABLE_CONTENT), + ( + {'given_name': 'Henry', 'pid': 'unknown_prefix:henry'}, + HTTP_422_UNPROCESSABLE_CONTENT, + ), ({'given_name': 'Henry', 'pid': 'xyz:henry'}, 200), ] diff --git a/dump_things_service/tests/test_web_interface.py b/dump_things_service/tests/test_web_interface.py index 0d435bb..74464e2 100644 --- a/dump_things_service/tests/test_web_interface.py +++ b/dump_things_service/tests/test_web_interface.py @@ -18,11 +18,11 @@ pids = ('', '--------', '&&&&&', 'abc', 'abc&', 'abc&format=ttl') tuple(product(*(collection_names, class_names, queries, format_names))), ) def test_web_interface_post_errors( - fastapi_client_simple, - collection_name, - class_name, - query, - format_name, + fastapi_client_simple, + collection_name, + class_name, + query, + format_name, ): """Check that no internal server error occurs with weird input""" test_client, _, _ = fastapi_client_simple @@ -39,11 +39,11 @@ def test_web_interface_post_errors( tuple(product(*(collection_names, class_names, queries, format_names))), ) def test_web_interface_get_class_errors( - fastapi_client_simple, - collection_name, - class_name, - query, - format_name, + fastapi_client_simple, + collection_name, + class_name, + query, + format_name, ): """Check that no internal server error occurs with weird input""" test_client, _, _ = fastapi_client_simple @@ -64,11 +64,11 @@ def test_web_interface_get_class_errors( tuple(product(*(collection_names, pids, queries, format_names))), ) def test_web_interface_get_pid_errors( - fastapi_client_simple, - collection_name, - pid, - query, - format_name, + fastapi_client_simple, + collection_name, + pid, + query, + format_name, ): """Check that no internal server error occurs with weird input""" test_client, _, _ = fastapi_client_simple diff --git a/dump_things_service/token_endpoints.py b/dump_things_service/token_endpoints.py index a45e1f2..3d650a4 100644 --- a/dump_things_service/token_endpoints.py +++ b/dump_things_service/token_endpoints.py @@ -70,11 +70,10 @@ def get_token_parts(token: str) -> list[str]: status_code=HTTP_201_CREATED, ) async def create_token( - response: Response, - body: TokenRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: TokenRequest, + api_key: str = Depends(api_key_header_scheme), ) -> TokenRequest: - token_request = create_or_replace_token(body, api_key, allow_replace=False) response.headers['Location'] = f'/tokens/{quote(body.name)}' return token_request @@ -87,23 +86,21 @@ async def create_token( status_code=HTTP_201_CREATED, ) async def replace_token( - response: Response, - body: TokenRequest, - api_key: str = Depends(api_key_header_scheme), + response: Response, + body: TokenRequest, + api_key: str = Depends(api_key_header_scheme), ) -> TokenRequest: - token_request = create_or_replace_token(body, api_key, allow_replace=True) response.headers['Location'] = f'/tokens/{quote(body.name)}' return token_request def create_or_replace_token( - body: TokenRequest, - api_key: str, - *, - allow_replace: bool, + body: TokenRequest, + api_key: str, + *, + allow_replace: bool, ) -> TokenRequest: - instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -117,7 +114,7 @@ def create_or_replace_token( ) # Ensure that all specified collections and modes exist - for collection_name, token_collection_info in body.collections.items(): + for collection_name, token_collection_info in body.collections.items(): if collection_name not in abstract_config.collections: detail = f"No such collection: '{collection_name}'." raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=detail) @@ -125,12 +122,11 @@ def create_or_replace_token( # Check that incoming areas are defined if the token allows writing. token_permissions = get_token_permissions(token_collection_info.mode) if token_permissions.incoming_write or token_permissions.zones_access: - # Check for incoming definition in collection config collection_info = abstract_config.collections[collection_name] if not collection_info.incoming: detail = ( - f"Cannot add token with write access to collection " + f'Cannot add token with write access to collection ' f"'{collection_name}' without `incoming`." ) raise HTTPException( @@ -153,7 +149,7 @@ def create_or_replace_token( token_representation=body.representation, ) if existing_token_info: - detail= "Token with identical representation already exists." + detail = 'Token with identical representation already exists.' raise HTTPException(status_code=HTTP_409_CONFLICT, detail=detail) else: # Generate a random representation that does not yet exist. @@ -202,9 +198,8 @@ def create_or_replace_token( name='Get existing tokens', ) async def get_tokens( - api_key: str = Depends(api_key_header_scheme), + api_key: str = Depends(api_key_header_scheme), ) -> list[TokenRequest]: - instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -228,10 +223,9 @@ async def get_tokens( name='Get token by name', ) async def get_token_with_name( - token_name: str, - api_key: str = Depends(api_key_header_scheme), + token_name: str, + api_key: str = Depends(api_key_header_scheme), ) -> TokenRequest: - instance_state = get_instance_state() abstract_config = get_config() @@ -258,10 +252,9 @@ async def get_token_with_name( name='Delete token with name', ) async def delete_token_with_name( - token_name: str, - api_key: str = Depends(api_key_header_scheme), + token_name: str, + api_key: str = Depends(api_key_header_scheme), ): - instance_state = get_instance_state() abstract_config = get_config() @@ -293,8 +286,8 @@ async def delete_token_with_name( status_code=HTTP_201_CREATED, ) async def create_admin_token( - body: AdminTokenRequest, - api_key: str = Depends(api_key_header_scheme), + body: AdminTokenRequest, + api_key: str = Depends(api_key_header_scheme), ): return create_or_replace_admin_token(body, api_key, allow_replace=False) @@ -306,17 +299,17 @@ async def create_admin_token( status_code=HTTP_201_CREATED, ) async def replace_admin_token( - body: AdminTokenRequest, - api_key: str = Depends(api_key_header_scheme), + body: AdminTokenRequest, + api_key: str = Depends(api_key_header_scheme), ): return create_or_replace_admin_token(body, api_key, allow_replace=True) def create_or_replace_admin_token( - body: AdminTokenRequest, - api_key: str, - *, - allow_replace: bool, + body: AdminTokenRequest, + api_key: str, + *, + allow_replace: bool, ): # Check for conflicting token-name if body.name == '__bootstrap__': @@ -327,11 +320,11 @@ def create_or_replace_admin_token( # Check for token content if not body.representation: - detail='Empty administrator token is not allowed' + detail = 'Empty administrator token is not allowed' raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) if not hash_matcher.match(body.representation.strip()): - detail='Hashed token is not a 64-digits hex-number' + detail = 'Hashed token is not a 64-digits hex-number' raise HTTPException(status_code=HTTP_406_NOT_ACCEPTABLE, detail=detail) instance_state = get_instance_state() @@ -368,7 +361,7 @@ def create_or_replace_admin_token( name='Get admin token names', ) async def get_admin_token( - api_key: str = Depends(api_key_header_scheme), + api_key: str = Depends(api_key_header_scheme), ) -> list[dict]: instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -376,10 +369,7 @@ async def get_admin_token( authenticate_admin(instance_state, abstract_config, api_key) return [ - { - 'name': token_name, - **(token_value.model_dump(mode='json', by_alias=True)) - } + {'name': token_name, **(token_value.model_dump(mode='json', by_alias=True))} for token_name, token_value in abstract_config.admin_tokens.items() ] + ( [] @@ -399,10 +389,9 @@ async def get_admin_token( name='Delete admin token with name', ) async def delete_admin_token( - token_name: str, - api_key: str = Depends(api_key_header_scheme), + token_name: str, + api_key: str = Depends(api_key_header_scheme), ): - instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index f40c903..0d6652b 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -6,6 +6,7 @@ To speed up processing, multiple indices could be introduced, e.g.: - token representation -> token name """ + from __future__ import annotations import logging @@ -95,7 +96,7 @@ def combine_ttl(documents: list[str]) -> str: def wrap_http_exception( exception_class: type[BaseException] = ValueError, status_code: int = HTTP_400_BAD_REQUEST, - header: str = '' + header: str = '', ): """Wrap exceptions of class `exception_class` into HTTP exceptions""" try: @@ -108,12 +109,11 @@ def wrap_http_exception( def join_default_token_permissions( - abstract_configuration: Configuration, - instance_state: InstanceState, - permissions: TokenPermission, - collection: str, + abstract_configuration: Configuration, + instance_state: InstanceState, + permissions: TokenPermission, + collection: str, ) -> TokenPermission: - result = permissions.model_copy() # Get the default token name. If a default token is not defined, return @@ -132,46 +132,41 @@ def join_default_token_permissions( if collection not in abstract_configuration.tokens[default_token_name].collections: return result - default_token_mode = abstract_configuration.tokens[default_token_name].collections[collection].mode + default_token_mode = ( + abstract_configuration.tokens[default_token_name].collections[collection].mode + ) default_token_permissions = mode_mapping[TokenModes(default_token_mode)] result.curated_read = ( - permissions.curated_read | default_token_permissions.curated_read + permissions.curated_read | default_token_permissions.curated_read ) result.incoming_read = ( - permissions.incoming_read | default_token_permissions.incoming_read + permissions.incoming_read | default_token_permissions.incoming_read ) result.incoming_write = ( - permissions.incoming_write | default_token_permissions.incoming_write + permissions.incoming_write | default_token_permissions.incoming_write ) return result def get_on_disk_labels( - store_path: Path, - abstract_config: Configuration, - collection: str, + store_path: Path, + abstract_config: Configuration, + collection: str, ) -> set[str]: check_collection(abstract_config, collection) - incoming_path = ( - store_path / abstract_config.collections[collection].incoming - ) + incoming_path = store_path / abstract_config.collections[collection].incoming if not incoming_path or not incoming_path.exists(): return set() - return { - path.name - for path in incoming_path.iterdir() - if path.is_dir() - } + return {path.name for path in incoming_path.iterdir() if path.is_dir()} def authenticate_token( - instance_state: InstanceState, - collection_name: str, - token_representation: str, + instance_state: InstanceState, + collection_name: str, + token_representation: str, ) -> AuthenticationInfo: - # Try to authenticate the token with the authentication providers that # are associated with the collection. auth_info = None @@ -204,9 +199,9 @@ def authenticate_token( def get_default_token_auth_info( - abstract_config: Configuration, - collection_name: str, - token_name: str, + abstract_config: Configuration, + collection_name: str, + token_name: str, ) -> AuthenticationInfo: token_config = abstract_config.tokens[token_name] collection_info = token_config.collections.get(collection_name) @@ -218,20 +213,19 @@ def get_default_token_auth_info( ) return AuthenticationInfo( token_permission=mode_mapping[TokenModes(collection_info.mode)], - user_id = token_config.user_id, - incoming_label = collection_info.incoming_label, + user_id=token_config.user_id, + incoming_label=collection_info.incoming_label, ) def get_token_store( - abstract_config: Configuration, - instance_state: InstanceState, - collection_name: str, - token_representation: str | None, - *, - is_token_name: bool = False, + abstract_config: Configuration, + instance_state: InstanceState, + collection_name: str, + token_representation: str | None, + *, + is_token_name: bool = False, ) -> tuple[_ModelStore, TokenPermission, str] | tuple[None, None, None, None]: - # If a token representation is provided, try to authenticate the token # with the authentication providers that are associated with the collection. if not is_token_name: @@ -277,11 +271,13 @@ def get_token_store( if not incoming: raise HTTPException( status_code=HTTP_401_UNAUTHORIZED, - detail='No incoming area for collection ' + collection_name + detail='No incoming area for collection ' + collection_name, ) # Check whether a store for this collection and token does already exist. - store_info = instance_state.incoming_stores[collection_name].get(token_representation) + store_info = instance_state.incoming_stores[collection_name].get( + token_representation + ) if store_info: return store_info @@ -302,11 +298,13 @@ def get_token_store( def create_store( - abstract_configuration: Configuration, - instance_state: InstanceState, - collection_name: str, + abstract_configuration: Configuration, + instance_state: InstanceState, + collection_name: str, ) -> _ModelStore: - collection_curated_path = abstract_configuration.collections[collection_name].curated + collection_curated_path = abstract_configuration.collections[ + collection_name + ].curated return create_token_store( abstract_configuration=abstract_configuration, instance_state=instance_state, @@ -316,10 +314,10 @@ def create_store( def create_token_store( - abstract_configuration: Configuration, - instance_state: InstanceState, - collection_name: str, - store_dir: Path, + abstract_configuration: Configuration, + instance_state: InstanceState, + collection_name: str, + store_dir: Path, ) -> _ModelStore: from dump_things_service.abstract_config import get_backend_and_extension from dump_things_service.backends.schema_type_layer import SchemaTypeLayer @@ -352,7 +350,6 @@ def create_token_store( backend_config = abstract_configuration.collections[collection_name].backend backend_name, extension = get_backend_and_extension(backend_config.type) if backend_name == 'record_dir': - backend = create_record_dir_token_store_backend( store_dir=store_dir, order_by=instance_state.order_by, @@ -374,7 +371,9 @@ def create_token_store( if extension == 'stl': backend = SchemaTypeLayer(backend=backend, schema=schema_uri) - submission_tags = abstract_configuration.collections[collection_name].submission_tags + submission_tags = abstract_configuration.collections[ + collection_name + ].submission_tags return ModelStore( schema=schema_uri, backend=backend, @@ -386,11 +385,11 @@ def create_token_store( def create_record_dir_token_store_backend( - store_dir: Path, - order_by: list[str], - schema_uri: str, - mapping_function: str, - suffix: str, + store_dir: Path, + order_by: list[str], + schema_uri: str, + mapping_function: str, + suffix: str, ) -> _RecordDirStore: from dump_things_service.backends.record_dir import RecordDirStore from dump_things_service.instance_state import record_dir_config_file_name @@ -414,15 +413,16 @@ def create_record_dir_token_store_backend( def write_record_dir_config( - path: Path, - mapping_function: str, - schema: str, + path: Path, + mapping_function: str, + schema: str, ): from dump_things_service.instance_state import record_dir_config_file_name record_dir_config_file_path = path / record_dir_config_file_name if not record_dir_config_file_path.exists(): - record_dir_config_file_path.write_text(f"""# RecordDir Config + record_dir_config_file_path.write_text( + f"""# RecordDir Config type: records version: 1 schema: {schema} @@ -433,9 +433,9 @@ idfx: {mapping_function} def create_sqlite_token_store_backend( - store_dir: Path, - order_by: list[str], -) -> _SQLiteBackend: + store_dir: Path, + order_by: list[str], +) -> _SQLiteBackend: from dump_things_service.backends.sqlite import SQLiteBackend from dump_things_service.backends.sqlite import ( record_file_name as sqlite_record_file_name, @@ -448,26 +448,22 @@ def create_sqlite_token_store_backend( def check_bounds( - length: int | None, - max_length: int, - collection: str, - alternative_url: str + length: int | None, max_length: int, collection: str, alternative_url: str ): if length > max_length: raise HTTPException( status_code=HTTP_413_CONTENT_TOO_LARGE, detail=f"Too many records found in collection '{collection}'. " - f'Please use pagination (/{collection}{alternative_url}).', + f'Please use pagination (/{collection}{alternative_url}).', ) async def process_token( - abstract_config: Configuration, - instance_state: InstanceState, - api_key: str | None, - collection: str, + abstract_config: Configuration, + instance_state: InstanceState, + api_key: str | None, + collection: str, ) -> tuple[TokenPermission, _ModelStore]: - if api_key is None: collection_config = get_collection_config_by_name(abstract_config, collection) token_store, token_permissions, user_id = get_token_store( @@ -492,9 +488,9 @@ async def process_token( # Check for maintenance mode if collection in instance_state.maintenance_mode: if not ( - final_permissions.curated_read - and final_permissions.curated_write - and final_permissions.zones_access + final_permissions.curated_read + and final_permissions.curated_write + and final_permissions.zones_access ): raise HTTPException( status_code=HTTP_503_SERVICE_UNAVAILABLE, @@ -510,8 +506,8 @@ async def process_token( def get_required_incoming_labels( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> set[str]: return set( map( @@ -522,20 +518,19 @@ def get_required_incoming_labels( def get_required_incoming_info( - abstract_config: Configuration, - collection_name: str, + abstract_config: Configuration, + collection_name: str, ) -> set[tuple[str, str]]: return { (token_name, this_collection_info.incoming_label) for token_name, token_info in abstract_config.tokens.items() for this_collection_name, this_collection_info in token_info.collections.items() - if this_collection_name == collection_name and mode_mapping[ - TokenModes(this_collection_info.mode) - ].incoming_write is True + if this_collection_name == collection_name + and mode_mapping[TokenModes(this_collection_info.mode)].incoming_write is True } def var_escape( - name: str, + name: str, ) -> str: return name.replace('_', '___').replace('-', '_0_') diff --git a/dump_things_service/validate.py b/dump_things_service/validate.py index eae50f2..7b9bdd9 100644 --- a/dump_things_service/validate.py +++ b/dump_things_service/validate.py @@ -33,15 +33,14 @@ from dump_things_service.utils import ( def validate_record( - collection: str, - data: BaseModel | str, - class_name: str, - model: Any, - input_format: Format, - _: bool, - api_key: str | None = Depends(api_key_header_scheme), + collection: str, + data: BaseModel | str, + class_name: str, + model: Any, + input_format: Format, + _: bool, + api_key: str | None = Depends(api_key_header_scheme), ) -> JSONResponse: - instance_state = get_instance_state() abstract_config = get_config() @@ -82,18 +81,30 @@ def validate_record( ) if input_format == Format.ttl: - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Conversion error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Conversion error', + ): json_object = FormatConverter( abstract_config.collections[collection].schema_location, input_format=Format.ttl, output_format=Format.json, ).convert(data, class_name) - with wrap_http_exception(ValidationError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValidationError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): TypeAdapter(getattr(model, class_name)).validate_python(json_object) else: # Try to convert it into TTL to detect potential errors before storing # the record - with wrap_http_exception(ValueError, status_code=HTTP_422_UNPROCESSABLE_CONTENT, header='Validation error'): + with wrap_http_exception( + ValueError, + status_code=HTTP_422_UNPROCESSABLE_CONTENT, + header='Validation error', + ): instance_state.validators[collection].validate(data) return JSONResponse(True) -- 2.52.0 From 18f365ee1a0873f7a1d19de096fffc32eac66cf4 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jun 2026 21:04:59 +0200 Subject: [PATCH 17/19] chore: ready for readthedocs-based doc builds --- .readthedocs.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..630c0c1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,31 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + path: . + extra_requirements: + - docs -- 2.52.0 From 27458bf13adfdfe76d89bccca78a941febc6762b Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 1 Jul 2026 09:45:22 +0200 Subject: [PATCH 18/19] chore: apply subset of unsafe fixes from `hatch check code` This commit applies a subset of the fixes that were applied via `hatch check code --unsafe-fixes --fix`. --- dump_things_service/audit/gitaudit.py | 17 +++++---- .../audit/tests/test_gitaudit.py | 2 +- dump_things_service/auth/forgejo.py | 5 ++- dump_things_service/authenticate.py | 2 +- dump_things_service/backends/record_dir.py | 3 +- dump_things_service/collection.py | 24 +++++++------ dump_things_service/collection_endpoints.py | 12 +++---- dump_things_service/commands/check_pids.py | 20 ++++------- dump_things_service/commands/hash_token.py | 2 +- dump_things_service/commands/upload_config.py | 5 ++- dump_things_service/converter.py | 4 +-- dump_things_service/curated.py | 8 ++--- dump_things_service/incoming.py | 35 +++---------------- dump_things_service/instance_state.py | 14 +++++--- dump_things_service/main.py | 23 ++++++------ dump_things_service/store/model_store.py | 2 +- dump_things_service/tests/test_basic.py | 2 +- .../tests/test_collection_administration.py | 5 +-- dump_things_service/tests/test_config.py | 4 +-- dump_things_service/tests/test_curated.py | 13 +++---- .../tests/test_extract_inline.py | 6 ++-- dump_things_service/tests/test_incoming.py | 15 ++------ dump_things_service/tests/test_modes.py | 2 +- dump_things_service/token_endpoints.py | 19 +++++----- dump_things_service/utils.py | 28 ++++++--------- dump_things_service/validate.py | 2 +- 26 files changed, 111 insertions(+), 163 deletions(-) diff --git a/dump_things_service/audit/gitaudit.py b/dump_things_service/audit/gitaudit.py index 8bf7123..a3b3289 100644 --- a/dump_things_service/audit/gitaudit.py +++ b/dump_things_service/audit/gitaudit.py @@ -28,7 +28,7 @@ from datalad_core.runners import ( call_git, ) -from . import AuditBackend +from dump_things_service.audit import AuditBackend index_file_name = 'gitaudit_index.log' @@ -68,7 +68,8 @@ class GitAuditBackend(AuditBackend): self.lock = Lock() self.last_flush_time = 0 if auto_flush_timeout < 1: - raise ValueError('auto_flush_timeout must be greater or equal to 1') + msg = 'auto_flush_timeout must be greater or equal to 1' + raise ValueError(msg) self.flushing_thread = FlushingThread(self, auto_flush_timeout) self.flushing_thread.start() self._init_repo() @@ -144,12 +145,10 @@ class GitAuditBackend(AuditBackend): .splitlines() ) # Get the log entry - log_line = tuple( - filter( + log_line = next(filter( lambda l: not l.startswith('+++') and l.startswith('+'), log_diff_lines, - ) - )[0][1:] + ))[1:] log_entry = json.loads(log_line) # Get the YAML diff @@ -301,7 +300,7 @@ class GitAuditBackend(AuditBackend): record_id: str, ) -> tuple[str, Path, Path]: base = hashlib.sha1(record_id.encode()).hexdigest() - dir_1, dir_2, name = base[0:3], base[3:6], base[6:] + dir_1, dir_2, _name = base[0:3], base[3:6], base[6:] location_dir = Path(dir_1) / Path(dir_2) return ( base, @@ -333,7 +332,7 @@ class GitAuditBackend(AuditBackend): self._rebuild_index() with open(self.index_path) as f: - self.index = set(line.strip() for line in f) + self.index = {line.strip() for line in f} def _add_to_index( self, @@ -357,7 +356,7 @@ class GitAuditBackend(AuditBackend): for line in tree_entries: if not line.endswith('.yaml'): continue - flag, object_type, object_hash, file_name = line.split(maxsplit=3) + _flag, _object_type, object_hash, _file_name = line.split(maxsplit=3) record = yaml.safe_load( call_git( ['show', object_hash], diff --git a/dump_things_service/audit/tests/test_gitaudit.py b/dump_things_service/audit/tests/test_gitaudit.py index 25ddf12..adb93b5 100644 --- a/dump_things_service/audit/tests/test_gitaudit.py +++ b/dump_things_service/audit/tests/test_gitaudit.py @@ -44,7 +44,7 @@ def test_gitaudit_basic(tmp_path_factory): # Check that the changes are reported changes = backend.get_audit_log(record_id) assert len(changes) == 4 - assert tuple(map(lambda e: e[0:2], changes.values())) == tuple( + assert tuple(e[0:2] for e in changes.values()) == tuple( (f'committer_{100 + i}@x.org', f'author_{i}@y.org') for i in range(4) ) diff --git a/dump_things_service/auth/forgejo.py b/dump_things_service/auth/forgejo.py index 6288192..c84c557 100644 --- a/dump_things_service/auth/forgejo.py +++ b/dump_things_service/auth/forgejo.py @@ -13,8 +13,8 @@ from __future__ import annotations import hashlib import logging import time -from collections.abc import Callable from functools import wraps +from typing import TYPE_CHECKING import requests from requests.exceptions import Timeout @@ -31,6 +31,9 @@ from dump_things_service.auth import ( InvalidTokenError, ) +if TYPE_CHECKING: + from collections.abc import Callable + logger = logging.getLogger('dump_things_service') # Timeout for requests diff --git a/dump_things_service/authenticate.py b/dump_things_service/authenticate.py index 1ecb585..f45388c 100644 --- a/dump_things_service/authenticate.py +++ b/dump_things_service/authenticate.py @@ -13,7 +13,6 @@ from dump_things_service.abstract_config import ( check_collection, read_config, ) -from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.instance_state import get_instance_state from dump_things_service.utils import ( @@ -21,6 +20,7 @@ from dump_things_service.utils import ( ) if TYPE_CHECKING: + from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends import StorageBackend from dump_things_service.store.model_store import _ModelStore diff --git a/dump_things_service/backends/record_dir.py b/dump_things_service/backends/record_dir.py index 67b94a0..40ef6c1 100644 --- a/dump_things_service/backends/record_dir.py +++ b/dump_things_service/backends/record_dir.py @@ -7,7 +7,6 @@ The disk-layout is described in . from __future__ import annotations import logging -from collections.abc import Callable from pathlib import Path from typing import ( TYPE_CHECKING, @@ -26,7 +25,7 @@ from dump_things_service.backends import ( from dump_things_service.backends.record_dir_index import RecordDirIndex if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable __all__ = [ diff --git a/dump_things_service/collection.py b/dump_things_service/collection.py index bab7728..87a31ec 100644 --- a/dump_things_service/collection.py +++ b/dump_things_service/collection.py @@ -230,7 +230,7 @@ def create_collection( active_classes -= set(collection_configuration.ignore_classes) instance_state.collections[collection_name] = InstanceStateCollectionInfo( active_classes=active_classes, - tag_info=dict(), + tag_info={}, ) # Create a validator for the collection @@ -355,15 +355,14 @@ def check_record_dir_compatibility( # A record_dir-directory is considered to be empty, if it contains no # files or only an record_dir-index file - files_in_dir = tuple(map(lambda dir_entry: dir_entry.name, os.scandir(store_path))) + files_in_dir = tuple(dir_entry.name for dir_entry in os.scandir(store_path)) if files_in_dir in ((), (index_file_name,)): return record_dir_config = get_record_dir_config(store_path) if record_dir_config.schema_location != schema: - raise ConfigCollisionError( - f"Existing backend uses a different schema: '{record_dir_config.schema_location}'" - ) + msg = f"Existing backend uses a different schema: '{record_dir_config.schema_location}'" + raise ConfigCollisionError(msg) stored_mapping_method = record_dir_config.idfx.value if stored_mapping_method != backend_config.mapping_method: @@ -377,7 +376,8 @@ def check_sqlite_compatibility( ): sqlite_db_path = Path(store_path / sqlite_db_filename) if not sqlite_db_path.exists(): - raise ConfigError('No sqlite database found in existing store') + msg = 'No sqlite database found in existing store' + raise ConfigError(msg) def check_git_audit_compatibility( @@ -403,9 +403,11 @@ def check_git_audit_compatibility( force_c_locale=True, ) except CommandError as ce: - raise ConfigError(f'No git repository in gitaudit-path: {audit_path}') from ce + msg = f'No git repository in gitaudit-path: {audit_path}' + raise ConfigError(msg) from ce if result.strip().lower() != 'true': - raise ConfigError(f'No bare git repository in gitaudit-path: {audit_path}') + msg = f'No bare git repository in gitaudit-path: {audit_path}' + raise ConfigError(msg) return @@ -430,7 +432,7 @@ def create_endpoint( instance_state.collections[collection_name].tag_info[tag_group] = tag_name # TODO: get schema_info from instance_state!? - model, classes, model_var_name = get_model_for_schema( + model, _classes, model_var_name = get_model_for_schema( collection_config.schema_location ) globals()[model_var_name] = model @@ -556,10 +558,10 @@ def delete_endpoint( ): from fastapi.routing import _IncludedRouter - remove_paths_set = set( + remove_paths_set = { f'/{collection_name}/{operation_path}/{class_name}' for class_name in active_classes - ) + } remove_indices = [ index diff --git a/dump_things_service/collection_endpoints.py b/dump_things_service/collection_endpoints.py index 225c40c..451d492 100644 --- a/dump_things_service/collection_endpoints.py +++ b/dump_things_service/collection_endpoints.py @@ -3,7 +3,7 @@ from pathlib import ( Path, PurePosixPath, ) -from typing import Literal +from typing import Annotated, Literal from urllib.parse import quote from fastapi import ( @@ -71,7 +71,7 @@ class CollectionRequest(CollectionConfig): async def create_collection( response: Response, body: CollectionRequest, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): await create_or_replace_collection(body, api_key, allow_replace=False) response.headers['Location'] = f'/collections/{quote(body.name)}' @@ -86,7 +86,7 @@ async def create_collection( async def replace_collection( response: Response, body: CollectionRequest, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): await create_or_replace_collection(body, api_key, allow_replace=True) response.headers['Location'] = f'/collections/{quote(body.name)}' @@ -164,7 +164,7 @@ async def create_or_replace_collection( name='Get existing collections', ) async def get_collections( - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> list[CollectionRequest]: instance_state = get_instance_state() abstract_config = get_config() @@ -189,7 +189,7 @@ async def get_collections( ) async def get_collection_with_name( collection_name: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> CollectionConfig: instance_state = get_instance_state() abstract_config = get_config() @@ -212,7 +212,7 @@ async def get_collection_with_name( ) async def delete_collection( collection_name: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): instance_state = get_instance_state() abstract_config = get_config() diff --git a/dump_things_service/commands/check_pids.py b/dump_things_service/commands/check_pids.py index 46f4c17..12a44a0 100644 --- a/dump_things_service/commands/check_pids.py +++ b/dump_things_service/commands/check_pids.py @@ -2,8 +2,8 @@ from __future__ import annotations import sys from argparse import ArgumentParser -from collections.abc import Iterable from pathlib import Path +from typing import TYPE_CHECKING from fastapi import FastAPI @@ -16,12 +16,16 @@ from dump_things_service.backends.sqlite import _SQLiteBackend from dump_things_service.exceptions import CurieResolutionError from dump_things_service.instance_state import create_instance_state from dump_things_service.manifest import manifest_configuration -from dump_things_service.store.model_store import _ModelStore from dump_things_service.utils import ( create_token_store, get_on_disk_labels, ) +if TYPE_CHECKING: + from collections.abc import Iterable + + from dump_things_service.store.model_store import _ModelStore + parser = ArgumentParser( prog='Check pids for resolvability', description='This command checks for pids that are in CURIE format and ' @@ -33,16 +37,6 @@ parser.add_argument( ) -def show_backend(model_store: _ModelStore): - backend = model_store.backend - if isinstance(backend, _SchemaTypeLayer): - backend = backend.backend - if isinstance(backend, _SQLiteBackend): - print(f'Checking: {backend.db_path}', file=sys.stderr) - else: - print(f'Checking: {backend.root}', file=sys.stderr) - - def check_pids_in_stores(stores: Iterable[_ModelStore]) -> int: result = 0 for store in stores: @@ -53,8 +47,6 @@ def check_pids_in_stores(stores: Iterable[_ModelStore]) -> int: store.pid_to_iri(pid) except CurieResolutionError: result += 1 - print(pid, store.get_uri()) - return result diff --git a/dump_things_service/commands/hash_token.py b/dump_things_service/commands/hash_token.py index b84c912..9ced0cb 100644 --- a/dump_things_service/commands/hash_token.py +++ b/dump_things_service/commands/hash_token.py @@ -22,7 +22,7 @@ def main(): arguments = parser.parse_args() token = arguments.token.strip() - if any(map(lambda s: s.isspace(), token)): + if any(s.isspace() for s in token): print('Whitespace are not allowed in token', file=sys.stderr, flush=True) return 1 diff --git a/dump_things_service/commands/upload_config.py b/dump_things_service/commands/upload_config.py index c215fe1..6955e96 100644 --- a/dump_things_service/commands/upload_config.py +++ b/dump_things_service/commands/upload_config.py @@ -165,7 +165,7 @@ def convert_config_1_to_config_2( } store_path = Path(store_path) if store_path else None - for collection_name, collection_config in old_configuration['collections'].items(): + for collection_config in old_configuration['collections'].values(): backend = collection_config.get('backend') if backend and backend['type'].startswith('sqlite'): collection_config['schema'] = backend['schema'] @@ -187,14 +187,13 @@ def convert_config_1_to_config_2( collection_config['default_token'] ] - new_configuration = { + return { 'type': 'collections', 'version': 2, 'tokens': new_tokens_dict, 'collections': old_configuration['collections'], 'admin_tokens': {}, } - return new_configuration def establish_configuration( diff --git a/dump_things_service/converter.py b/dump_things_service/converter.py index c92fbb5..c39686a 100644 --- a/dump_things_service/converter.py +++ b/dump_things_service/converter.py @@ -1,7 +1,6 @@ from __future__ import annotations import re -from collections.abc import Callable from functools import cache from json import loads as json_loads from typing import ( @@ -13,7 +12,6 @@ from linkml.utils.datautils import ( get_dumper, get_loader, ) -from linkml_runtime import SchemaView from rdflib.term import ( URIRef, _toPythonMapping, @@ -30,8 +28,10 @@ from dump_things_service.model import ( from dump_things_service.utils import cleaned_json if TYPE_CHECKING: + from collections.abc import Callable from types import ModuleType + from linkml_runtime import SchemaView from pydantic import BaseModel from dump_things_service.backends import RecordInfo diff --git a/dump_things_service/curated.py b/dump_things_service/curated.py index b1e3237..f8a8ede 100644 --- a/dump_things_service/curated.py +++ b/dump_things_service/curated.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated from fastapi import ( APIRouter, @@ -24,7 +24,6 @@ from dump_things_service.abstract_config import ( read_config, ) from dump_things_service.api_key import api_key_header_scheme -from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer from dump_things_service.exceptions import CurieResolutionError from dump_things_service.instance_state import get_instance_state @@ -39,6 +38,7 @@ from dump_things_service.utils import ( if TYPE_CHECKING: from pydantic import BaseModel + from dump_things_service.auth import AuthenticationInfo from dump_things_service.backends import StorageBackend from dump_things_service.lazy_list import LazyList from dump_things_service.store.model_store import _ModelStore @@ -176,7 +176,7 @@ async def read_curated_all_records_paginated( async def read_curated_record_with_pid( collection: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _read_curated_records( collection=collection, @@ -194,7 +194,7 @@ async def read_curated_record_with_pid( async def delete_curated_record_with_pid( collection: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _delete_curated_record( collection=collection, diff --git a/dump_things_service/incoming.py b/dump_things_service/incoming.py index 53a5b46..621c652 100644 --- a/dump_things_service/incoming.py +++ b/dump_things_service/incoming.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated from fastapi import ( APIRouter, @@ -59,7 +59,7 @@ add_pagination(router) ) async def incoming_read_labels( collection: str, - api_key: str | None = Depends(api_key_header_scheme), + api_key: Annotated[str | None, Depends(api_key_header_scheme)], ) -> list[str]: # Authorize api_key await authorize_zones(collection, api_key) @@ -186,7 +186,7 @@ async def incoming_read_record_with_pid( collection: str, label: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _incoming_read_records( collection=collection, @@ -206,7 +206,7 @@ async def incoming_delete_record_with_pid( collection: str, label: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return await _incoming_delete_record( collection=collection, @@ -300,33 +300,6 @@ async def _get_store_and_backend( store_dir=store_dir, ) - xxx = """ - # For consistency, associate the store with all matching tokens from the - # configuration file. That means with all tokens that have the same - # input - matching_tokens = [ - token_name - for token_name, token_info in abstract_config.tokens.items() - if (collection, label) in [ - (collection_name, token_collection_info.incoming_label) - for collection_name, token_collection_info in token_info.items() - ] - ] - - for matching_token in matching_tokens: - # Associate the store with all matching tokens in the configuration. - # Note: there are stores that are not associated with a token in - # the abstract configuration. These are stores that belong to a token - # that is authenticated with an external authentication source. - token_info = instance_state.tokens[collection][matching_token] - instance_state.token_stores[collection][matching_token] = ( - model_store, - matching_token, - token_info['permissions'], - token_info['user_id'], - ) - """ - backend = model_store.backend if isinstance(backend, _SchemaTypeLayer): return model_store, backend.backend diff --git a/dump_things_service/instance_state.py b/dump_things_service/instance_state.py index c41dbc3..703febb 100644 --- a/dump_things_service/instance_state.py +++ b/dump_things_service/instance_state.py @@ -2,17 +2,13 @@ from __future__ import annotations import dataclasses import logging -from collections.abc import Callable from functools import cache -from pathlib import Path -from types import ModuleType from typing import ( + TYPE_CHECKING, Any, ) import yaml -from fastapi import FastAPI -from linkml_runtime import SchemaView from pydantic import ValidationError from yaml.scanner import ScannerError @@ -29,6 +25,14 @@ from dump_things_service.model import ( get_schema_view, ) +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from types import ModuleType + + from fastapi import FastAPI + from linkml_runtime import SchemaView + logger = logging.getLogger('dump_things_service') record_dir_config_file_name = '.dumpthings.yaml' diff --git a/dump_things_service/main.py b/dump_things_service/main.py index 63f0f51..b357563 100644 --- a/dump_things_service/main.py +++ b/dump_things_service/main.py @@ -5,7 +5,7 @@ import logging import os import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated from dump_things_service.abstract_config import store_config from dump_things_service.commands.upload_config import convert_config_1_to_config_2 @@ -309,15 +309,14 @@ if not ( g_configuration.admin_tokens or g_configuration.collections or g_configuration.tokens -): - if not g_instance_state.bootstrap_token: - print( - 'The server has an empty configuration and requires a bootstrap ' - 'token (use `--admin-token-hash` to provide one).', - file=sys.stderr, - flush=True, - ) - sys.exit(2) +) and not g_instance_state.bootstrap_token: + print( + 'The server has an empty configuration and requires a bootstrap ' + 'token (use `--admin-token-hash` to provide one)', + file=sys.stderr, + flush=True, + ) + sys.exit(2) manifest_configuration( @@ -360,7 +359,7 @@ async def server() -> ServerResponse: ) async def maintenance( body: MaintenanceRequest, - api_key: str | None = Depends(api_key_header_scheme), + api_key: Annotated[str | None, Depends(api_key_header_scheme)], ): if api_key is None: raise HTTPException( @@ -666,7 +665,7 @@ async def _read_records_of_type( async def delete_record( collection: str, pid: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): check_collection(g_configuration, collection) final_permissions, token_store = await process_token( diff --git a/dump_things_service/store/model_store.py b/dump_things_service/store/model_store.py index 489a3b5..61600a5 100644 --- a/dump_things_service/store/model_store.py +++ b/dump_things_service/store/model_store.py @@ -43,7 +43,7 @@ class _ModelStore: submitter: str | None, ) -> Iterable[tuple[str, dict]]: if obj.__class__.__name__ == 'Thing' and 'dlthings:placeholder' in ( - obj.annotations or dict() + obj.annotations or {} ): return [] diff --git a/dump_things_service/tests/test_basic.py b/dump_things_service/tests/test_basic.py index 7c9c966..5e88c02 100644 --- a/dump_things_service/tests/test_basic.py +++ b/dump_things_service/tests/test_basic.py @@ -293,7 +293,7 @@ def test_funky_pid(fastapi_client_simple): def test_token_store_priority(fastapi_client_simple): - test_client, store_dir, _ = fastapi_client_simple + test_client, _store_dir, _ = fastapi_client_simple # Post a record with the same pid as the global store's test record, but # with different content. diff --git a/dump_things_service/tests/test_collection_administration.py b/dump_things_service/tests/test_collection_administration.py index 331efbb..6e9a943 100644 --- a/dump_things_service/tests/test_collection_administration.py +++ b/dump_things_service/tests/test_collection_administration.py @@ -68,10 +68,7 @@ def _name_in_openapi_paths( ) -> bool: response = test_client.get('/openapi.json') open_api = response.json() - for path in open_api['paths'].keys(): - if name in path: - return True - return False + return any(name in path for path in open_api['paths']) def test_collection_adding(fastapi_client_simple): diff --git a/dump_things_service/tests/test_config.py b/dump_things_service/tests/test_config.py index 2cc88d5..1d2c30d 100644 --- a/dump_things_service/tests/test_config.py +++ b/dump_things_service/tests/test_config.py @@ -77,7 +77,7 @@ def test_collection_dir_reuse_detection(fastapi_client_simple): def test_scanner_error_detection(tmp_path_factory): tmp_path = tmp_path_factory.mktemp('config_scanner_test') - config_backend, audit_backend = get_config_backends(tmp_path) + config_backend, _audit_backend = get_config_backends(tmp_path) config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', @@ -96,7 +96,7 @@ def test_scanner_error_detection(tmp_path_factory): def test_structure_error_detection(tmp_path_factory): tmp_path = tmp_path_factory.mktemp('config_scanner_test') - config_backend, audit_backend = get_config_backends(tmp_path) + config_backend, _audit_backend = get_config_backends(tmp_path) config_backend.add_record( iri=dump_things_config_iri, class_name='DumpThingsConfig', diff --git a/dump_things_service/tests/test_curated.py b/dump_things_service/tests/test_curated.py index 8ae7ff5..7e121ec 100644 --- a/dump_things_service/tests/test_curated.py +++ b/dump_things_service/tests/test_curated.py @@ -19,8 +19,8 @@ delete_record = { } -@pytest.mark.parametrize('paginate', ('', 'p/')) -@pytest.mark.parametrize('class_name', ('', 'Person')) +@pytest.mark.parametrize('paginate', ['', 'p/']) +@pytest.mark.parametrize('class_name', ['', 'Person']) def test_read_curated_records( fastapi_client_simple, paginate, @@ -54,12 +54,6 @@ def test_read_curated_records( assert len(json_object) == count -pytest.mark.parametrize( - 'pid', - ('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'), -) - - def test_read_curated_records_by_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple @@ -187,5 +181,6 @@ def test_audit_backend_auto_flush(fastapi_client_simple): break i += 1 if i == 10: - raise ValueError('auto flush did not trigger within 10 seconds') + msg = 'auto flush did not trigger within 10 seconds' + raise ValueError(msg) time.sleep(1) diff --git a/dump_things_service/tests/test_extract_inline.py b/dump_things_service/tests/test_extract_inline.py index d88be3c..5501174 100644 --- a/dump_things_service/tests/test_extract_inline.py +++ b/dump_things_service/tests/test_extract_inline.py @@ -348,7 +348,7 @@ def _check_result_json( # That breaks the tests. They assume that Person.relations has range Thing. @pytest.mark.xfail def test_dont_extract_empty_things_on_service(fastapi_client_simple): - test_client, store = fastapi_client_simple + test_client, _store = fastapi_client_simple for i in range(1, 3): # Deposit JSON record @@ -361,7 +361,7 @@ def test_dont_extract_empty_things_on_service(fastapi_client_simple): def test_store_things(fastapi_client_simple): - test_client, store, _ = fastapi_client_simple + test_client, _store, _ = fastapi_client_simple simple_thing = { 'pid': 'http://test.simple.thing/1', @@ -384,7 +384,7 @@ def test_store_things(fastapi_client_simple): def test_store_complex_things(fastapi_client_simple): - test_client, store, _ = fastapi_client_simple + test_client, _store, _ = fastapi_client_simple complex_thing = { 'pid': 'http://test.complex.thing/1', diff --git a/dump_things_service/tests/test_incoming.py b/dump_things_service/tests/test_incoming.py index ecc7cad..456cc69 100644 --- a/dump_things_service/tests/test_incoming.py +++ b/dump_things_service/tests/test_incoming.py @@ -61,8 +61,8 @@ def fill_zones(test_client): zones_filled = True -@pytest.mark.parametrize('paginate', ('', 'p/')) -@pytest.mark.parametrize('class_name', ('', 'Person')) +@pytest.mark.parametrize('paginate', ['', 'p/']) +@pytest.mark.parametrize('class_name', ['', 'Person']) def test_read_incoming_records( fastapi_client_simple, paginate: str, @@ -106,10 +106,7 @@ def test_read_incoming_records( ) assert response.status_code == HTTP_200_OK json_object = response.json() - if 'items' in json_object: - result = json_object['items'] - else: - result = json_object + result = json_object['items'] if 'items' in json_object else json_object matching = [ json_object for json_object in result if json_object['pid'] == pattern ] @@ -118,12 +115,6 @@ def test_read_incoming_records( ) -pytest.mark.parametrize( - 'pid', - ('abc:mode_test', 'abc:some_timee@x.com', 'abc:curated'), -) - - def test_read_incoming_records_by_pid(fastapi_client_simple): test_client, _, _ = fastapi_client_simple diff --git a/dump_things_service/tests/test_modes.py b/dump_things_service/tests/test_modes.py index 2b22206..07afa32 100644 --- a/dump_things_service/tests/test_modes.py +++ b/dump_things_service/tests/test_modes.py @@ -50,7 +50,7 @@ def verify_modes( def test_token_modes(fastapi_client_simple): - test_client, store_dir, _ = fastapi_client_simple + test_client, _store_dir, _ = fastapi_client_simple # Post a record to incoming of collections `collection_1`. We use it to # validate read/write permissions on class-base diff --git a/dump_things_service/token_endpoints.py b/dump_things_service/token_endpoints.py index 3d650a4..c7e6b0f 100644 --- a/dump_things_service/token_endpoints.py +++ b/dump_things_service/token_endpoints.py @@ -1,6 +1,7 @@ import logging import random import re +from typing import Annotated from urllib.parse import quote from fastapi import ( @@ -72,7 +73,7 @@ def get_token_parts(token: str) -> list[str]: async def create_token( response: Response, body: TokenRequest, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> TokenRequest: token_request = create_or_replace_token(body, api_key, allow_replace=False) response.headers['Location'] = f'/tokens/{quote(body.name)}' @@ -88,7 +89,7 @@ async def create_token( async def replace_token( response: Response, body: TokenRequest, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> TokenRequest: token_request = create_or_replace_token(body, api_key, allow_replace=True) response.headers['Location'] = f'/tokens/{quote(body.name)}' @@ -198,7 +199,7 @@ def create_or_replace_token( name='Get existing tokens', ) async def get_tokens( - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> list[TokenRequest]: instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -224,7 +225,7 @@ async def get_tokens( ) async def get_token_with_name( token_name: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> TokenRequest: instance_state = get_instance_state() abstract_config = get_config() @@ -253,7 +254,7 @@ async def get_token_with_name( ) async def delete_token_with_name( token_name: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): instance_state = get_instance_state() abstract_config = get_config() @@ -287,7 +288,7 @@ async def delete_token_with_name( ) async def create_admin_token( body: AdminTokenRequest, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return create_or_replace_admin_token(body, api_key, allow_replace=False) @@ -300,7 +301,7 @@ async def create_admin_token( ) async def replace_admin_token( body: AdminTokenRequest, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): return create_or_replace_admin_token(body, api_key, allow_replace=True) @@ -361,7 +362,7 @@ def create_or_replace_admin_token( name='Get admin token names', ) async def get_admin_token( - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ) -> list[dict]: instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) @@ -390,7 +391,7 @@ async def get_admin_token( ) async def delete_admin_token( token_name: str, - api_key: str = Depends(api_key_header_scheme), + api_key: Annotated[str, Depends(api_key_header_scheme)], ): instance_state = get_instance_state() abstract_config = read_config(store_path=instance_state.store_path) diff --git a/dump_things_service/utils.py b/dump_things_service/utils.py index 0d6652b..97d4e5e 100644 --- a/dump_things_service/utils.py +++ b/dump_things_service/utils.py @@ -474,7 +474,7 @@ async def process_token( is_token_name=True, ) else: - token_store, token_permissions, user_id = get_token_store( + token_store, token_permissions, _user_id = get_token_store( abstract_config, instance_state, collection, @@ -486,16 +486,15 @@ async def process_token( ) # Check for maintenance mode - if collection in instance_state.maintenance_mode: - if not ( - final_permissions.curated_read - and final_permissions.curated_write - and final_permissions.zones_access - ): - raise HTTPException( - status_code=HTTP_503_SERVICE_UNAVAILABLE, - detail=f"Collection '{collection}' is in maintenance mode", - ) + if collection in instance_state.maintenance_mode and not ( + final_permissions.curated_read + and final_permissions.curated_write + and final_permissions.zones_access + ): + raise HTTPException( + status_code=HTTP_503_SERVICE_UNAVAILABLE, + detail=f"Collection '{collection}' is in maintenance mode", + ) if not final_permissions.incoming_read and not final_permissions.curated_read: raise HTTPException( @@ -509,12 +508,7 @@ def get_required_incoming_labels( abstract_config: Configuration, collection_name: str, ) -> set[str]: - return set( - map( - lambda x: x[1], - get_required_incoming_info(abstract_config, collection_name), - ) - ) + return {x[1] for x in get_required_incoming_info(abstract_config, collection_name)} def get_required_incoming_info( diff --git a/dump_things_service/validate.py b/dump_things_service/validate.py index 7b9bdd9..4f80e31 100644 --- a/dump_things_service/validate.py +++ b/dump_things_service/validate.py @@ -62,7 +62,7 @@ def validate_record( else api_key ) - store, token_permissions, user_id = get_token_store( + _store, token_permissions, _user_id = get_token_store( abstract_config, instance_state, collection, -- 2.52.0 From 79750f4381fa294a857e19f5bba97a9434237574 Mon Sep 17 00:00:00 2001 From: Christian Monch Date: Wed, 1 Jul 2026 09:54:10 +0200 Subject: [PATCH 19/19] fix: don't report token in error messages Ensure that an invalid token is not reported in error messages from Forgejo-authentication modules. --- dump_things_service/auth/forgejo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dump_things_service/auth/forgejo.py b/dump_things_service/auth/forgejo.py index c84c557..72c018e 100644 --- a/dump_things_service/auth/forgejo.py +++ b/dump_things_service/auth/forgejo.py @@ -141,7 +141,8 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache): ) from e if r.status_code >= HTTP_300_MULTIPLE_CHOICES: - msg = f'invalid token: ({r.status_code}): {r.text}' + cleaned_text = r.text.replace(token, '***') + msg = f'invalid token: ({r.status_code}): {cleaned_text}' raise InvalidTokenError(msg) return r.json() -- 2.52.0