orinoco/dump-things-service
orinoco/dump-things-service
Watch
5
0
Fork
You've already forked dump-things-service
 1
Code Issues 38 Pull requests Packages Activity Actions

Pass all code and format checks #249

Merged
cmo merged 5 commits from pass-code-check into master 2026-07-01 15:31:49 +00:00
Conversation 0 Commits 5 Files changed 31 +209 -155
31 changed files with 209 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

7
dump_things_service/__init__.py
View file

@ -1,8 +1,5 @@
from enum import Enum
from typing import (
Any,
Union,
)
from typing import Any
from starlette.status import (
HTTP_200_OK,
@ -50,7 +47,7 @@ class Format(str, Enum):
ttl = 'ttl'
JSON = Union[dict[str, Any], list[Any], str, int, float, None]
JSON = dict[str, Any] | list[Any] | str | int | float | None
YAML = JSON
config_file_name = '.dumpthings.yaml'

16
dump_things_service/abstract_config.py
View file

@ -43,6 +43,9 @@ config_audit_path = dump_things_private_path / 'config_audit'
config_backend = None
config_audit = None
CONFIG_VERSION_1_ID = 1
CONFIG_VERSION_2_ID = 2
class StrictModel(BaseModel):
model_config = ConfigDict(
@ -203,8 +206,8 @@ def get_token_permissions(mode: str) -> TokenPermission:
def get_config_backends(
store_path: Path,
) -> tuple[_RecordDirStore, GitAuditBackend]:
global config_audit
global config_backend
global config_audit # noqa PLW0603 -- this is cached on the first call
global config_backend # noqa PLW0603 -- this is cached on the first call
config_path = store_path / config_backend_path
if not config_path.exists():
@ -226,9 +229,10 @@ def get_config_backends(
def read_config(
store_path: Path,
*,
force_reload: bool = False,
) -> Configuration:
global g_abstract_configuration
global g_abstract_configuration # noqa PLW0603 -- this is cached on the first call
if not g_abstract_configuration or force_reload:
config_backend, _ = get_config_backends(store_path)
@ -253,8 +257,6 @@ def read_config(
def get_config() -> Configuration:
global g_abstract_configuration
if not g_abstract_configuration:
msg = 'Configuration not yet loaded'
raise RuntimeError(msg)
@ -265,7 +267,7 @@ def store_config(
store_path,
config: Configuration,
):
global g_abstract_configuration
global g_abstract_configuration # noqa PLW0603 -- this function updates a globally referenced instance
config_backend, audit_backend = get_config_backends(store_path)
json_object = config.model_dump(mode='json', exclude_none=True, by_alias=True)
@ -310,7 +312,7 @@ def check_label(
collection: str,
label: str,
):
from dump_things_service.utils import get_on_disk_labels
from dump_things_service.utils import get_on_disk_labels # noqa PLC0415 -- global import leads to circular imports
"""Check that a label exists in a collection configuration or on disk"""
if label not in get_config_labels(

29
dump_things_service/audit/gitaudit.py
View file

@ -13,7 +13,10 @@ import hashlib
import json
import re
import time
from datetime import datetime
from datetime import (
UTC,
datetime,
)
from pathlib import Path
from threading import (
Lock,
@ -32,6 +35,8 @@ from dump_things_service.audit import AuditBackend
index_file_name = 'gitaudit_index.log'
GIT_ERROR_UNCLEAN_EXIT = 128
class FlushingThread(Thread):
def __init__(
@ -145,10 +150,12 @@ class GitAuditBackend(AuditBackend):
.splitlines()
)
# Get the log entry
log_line = next(filter(
lambda l: not l.startswith('+++') and l.startswith('+'),
log_line = next(
filter(
lambda line: not line.startswith('+++') and line.startswith('+'),
log_diff_lines,
))[1:]
)
)[1:]
log_entry = json.loads(log_line)
# Get the YAML diff
@ -161,7 +168,9 @@ class GitAuditBackend(AuditBackend):
.decode()
.splitlines()
)
yaml_diff = '\n'.join(filter(lambda l: l != '', yaml_diff_lines)) + '\n'
yaml_diff = (
'\n'.join(filter(lambda line: line != '', yaml_diff_lines)) + '\n'
)
# Get the YAML content
yaml_content = call_git(
@ -233,7 +242,7 @@ class GitAuditBackend(AuditBackend):
committer_id: str,
author_id: str,
) -> None:
time_stamp = datetime.now().isoformat()
time_stamp = datetime.now(tz=UTC).isoformat()
entry = {
'time_stamp': time_stamp,
'committer_id': committer_id,
@ -262,7 +271,7 @@ class GitAuditBackend(AuditBackend):
capture_output=True,
)
except CommandError as e:
if e.returncode == 128:
if e.returncode == GIT_ERROR_UNCLEAN_EXIT:
return b''
raise
@ -299,7 +308,7 @@ class GitAuditBackend(AuditBackend):
self,
record_id: str,
) -> tuple[str, Path, Path]:
base = hashlib.sha1(record_id.encode()).hexdigest()
base = hashlib.sha1(record_id.encode()).hexdigest() # noqa S324 -- hash is not used for security
dir_1, dir_2, _name = base[0:3], base[3:6], base[6:]
location_dir = Path(dir_1) / Path(dir_2)
return (
@ -329,7 +338,7 @@ class GitAuditBackend(AuditBackend):
self.repo = Repo(self.path)
if not self.index_path.exists():
self._rebuild_index()
self.rebuild_index()
with open(self.index_path) as f:
self.index = {line.strip() for line in f}
@ -342,7 +351,7 @@ class GitAuditBackend(AuditBackend):
self.cached_index_entries.append(record_id)
self.index.add(record_id)
def _rebuild_index(self):
def rebuild_index(self):
tree_entries = (
call_git(
['ls-tree', '-r', 'master:'],

23
dump_things_service/audit/tests/test_gitaudit.py
View file

@ -6,7 +6,7 @@ from dump_things_service.audit.gitaudit import GitAuditBackend
def _get_git_log(path: Path) -> list[str]:
result = subprocess.run(
['git', '-C', str(path), 'log', '--oneline'],
['git', '-C', str(path), 'log', '--oneline'], # noqa S607 -- test run in a controlled environment
capture_output=True,
check=True,
)
@ -25,25 +25,26 @@ def test_gitaudit_basic(tmp_path_factory):
record_id = 'test_gitaudit_basic'
for index in range(4):
entry_count = 4
for index in range(entry_count):
backend.add_record(
record={'pid': record_id, 'content': index},
committer_id=f'committer_{100 + index}@x.org',
author_id=f'author_{index}@y.org',
)
# Check that the log file has 4 entries
# Check that the log file has `entry_count` entries
backend.flush()
log_lines = _get_audit_log_lines(backend, record_id)
assert len(log_lines) == 4
assert len(log_lines) == entry_count
# Check that the commit log has 4 + 1 (from `README.txt`) entries
# Check that the commit log has `entry_count` + 1 (from `README.txt`) entries
commit_log_lines = _get_git_log(tmp_path)
assert len(commit_log_lines) == 5
assert len(commit_log_lines) == entry_count + 1
# Check that the changes are reported
changes = backend.get_audit_log(record_id)
assert len(changes) == 4
assert len(changes) == entry_count
assert tuple(e[0:2] for e in changes.values()) == tuple(
(f'committer_{100 + i}@x.org', f'author_{i}@y.org') for i in range(4)
)
@ -67,18 +68,20 @@ def test_gitaudit_identical_change(tmp_path_factory):
author_id='author_b@y.org',
)
expected_entries = 1
# Check that there is only one entry in the audit log
log_lines = _get_audit_log_lines(backend, record_id)
assert len(log_lines) == 1
assert len(log_lines) == expected_entries
# Check that there are two entries in the commit history, one for the
# `README.txt`-file, one for the log entries.
commit_log_lines = _get_git_log(tmp_path)
assert len(commit_log_lines) == 2
assert len(commit_log_lines) == expected_entries + 1
# Check that the changes are reported
changes = backend.get_audit_log(record_id)
assert len(changes) == 1
assert len(changes) == expected_entries
def test_gitaudit_huge_log(tmp_path_factory):

22
dump_things_service/auth/forgejo.py
View file

@ -209,7 +209,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
def _get_unit_content(self, team: dict, unit_name: str) -> str:
permissions = team['units_map'].get(unit_name)
if not permissions:
logger.debug(f'no unit `repo.actions` in team {self.team}')
logger.debug('no unit `repo.actions` in team %s', self.team)
msg = (
f'no `repo.{unit_name}`-unit defined for team `{self.team}` in '
f'organization {self.organization}'
@ -221,7 +221,7 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
return permissions
def _instance_label(self) -> str:
return self.instance_id or hashlib.md5(self.api_url.encode()).hexdigest()
return self.instance_id or hashlib.md5(self.api_url.encode()).hexdigest() # noqa S324 -- hash not used for cryptography, only for label generation
@MethodCache.cache_temporary(duration=60)
def authenticate(
@ -229,14 +229,17 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
token: str,
) -> AuthenticationInfo:
logger.debug(
f'starting Forgejo authentication: {self.api_url}, {self.organization}, {self.team}'
'starting Forgejo authentication: %s, %s, %s',
self.api_url,
self.organization,
self.team,
)
user_teams = self._get_teams_for_user(token)
logger.debug(f'user_teams: {user_teams}')
logger.debug('user_teams: %s', str(user_teams))
if self.team not in user_teams:
logger.debug(f"{self.team} not in user's teams")
logger.debug("%s not in user's teams", self.team)
msg = f'token user is not member of team `{self.team}`'
raise RemoteAuthenticationError(
status=HTTP_401_UNAUTHORIZED,
@ -257,12 +260,12 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
token,
self.organization,
)
logger.debug(f'organization_teams: {organization_teams}')
logger.debug('organization_teams: %s', str(organization_teams))
# Check that the configured team exists
team = organization_teams.get(self.team)
if not team:
logger.debug(f'{self.team} not in organization teams')
logger.debug('%s not in organization teams', self.team)
if self.repository is not None:
msg = f'team `{self.team}` has no access to repository `{self.repository}`'
else:
@ -276,8 +279,9 @@ class ForgejoAuthenticationSource(AuthenticationSource, MethodCache):
code_permissions = self._get_unit_content(team, 'repo.code')
action_permissions = self._get_unit_content(team, 'repo.actions')
logger.debug(
f'authentication success, team permissions: {code_permissions}, '
f'{action_permissions}'
'authentication success, team permissions: %s, %s',
code_permissions,
action_permissions,
)
return AuthenticationInfo(
token_permission=self._get_permissions(

4
dump_things_service/backends/record_dir.py
View file

@ -164,7 +164,7 @@ class _RecordDirStore(StorageBackend):
def get_records_of_classes(
self,
class_names: list[str],
pattern: str | None = None,
_pattern: str | None = None,
) -> RecordDirResultList:
return RecordDirResultList().add_info(
sorted(
@ -184,7 +184,7 @@ class _RecordDirStore(StorageBackend):
def get_all_records(
self,
pattern: str | None = None,
_pattern: str | None = None,
) -> RecordDirResultList:
return RecordDirResultList().add_info(
sorted(

14
dump_things_service/backends/sqlite.py
View file

@ -130,9 +130,13 @@ class _SQLiteBackend(StorageBackend):
order_by: Iterable[str] | None = None,
echo: bool = False,
) -> None:
assert db_path.is_absolute(), f'db_path not absolute {db_path}'
if db_path.exists():
assert db_path.is_file(), f'db_path not a file {db_path}'
if not db_path.is_absolute():
msg = f'db_path not absolute: {db_path}'
raise ValueError(msg)
if db_path.exists() and not db_path.is_file():
msg = f'db_path not a file: {db_path}'
raise ValueError(msg)
super().__init__(order_by=order_by)
self.db_path = db_path
self.perform_file_name_conversion()
@ -243,14 +247,14 @@ class _SQLiteBackend(StorageBackend):
class_list = ', '.join(f"'{cn}'" for cn in class_names)
if pattern is None:
statement = text(
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id '
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' # noqa S608 -- all variables are controlled
'from thing '
f'where thing.class_name in ({class_list}) '
'ORDER BY thing.sort_key'
)
else:
statement = text(
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id '
'select distinct thing.iri, thing.class_name, thing.sort_key, thing.id ' # noqa S608 -- all variables are controlled
'from thing, json_tree(thing.object) '
'where lower(json_tree.value) like lower(:pattern) '
f'and thing.class_name in ({class_list}) '

25
dump_things_service/collection.py
View file

@ -19,6 +19,7 @@ from fastapi import (
FastAPI,
HTTPException,
)
from fastapi.routing import _IncludedRouter
from pydantic import (
BaseModel,
TypeAdapter,
@ -96,7 +97,17 @@ def {name}(
format: Format = Format.json,
) -> JSONResponse | PlainTextResponse:
logger.info('{name}(%s, %s, %s, %s, %s)', repr(data), repr('{class_name}'), repr({model_var_name}), repr(add_submission_tag), repr(format))
return {handler}('{collection}', data, '{class_name}', {model_var_name}, format, add_submission_tag, api_key)
return {handler}('{collection}', data, '{class_name}', {model_var_name}, format, api_key, add_submission_tag=add_submission_tag)
"""
_endpoint_validate_template = """
def {name}(
data: {model_var_name}.{class_name} | Annotated[str, Body(media_type='text/plain')],
api_key: str = Depends(api_key_header_scheme),
format: Format = Format.json,
) -> JSONResponse | PlainTextResponse:
logger.info('{name}(%s, %s, %s, %s)', repr(data), repr('{class_name}'), repr({model_var_name}), repr(format))
return {handler}('{collection}', data, '{class_name}', {model_var_name}, format, api_key)
"""
_endpoint_curated_template = """
@ -305,7 +316,9 @@ def write_record_dir_config(
backend_config: RecordDirBackendConfig,
schema: str,
):
assert isinstance(backend_config, RecordDirBackendConfig)
if not isinstance(backend_config, RecordDirBackendConfig):
msg = 'backend_config is not an instance of RecordDirBackendConfig'
raise TypeError(msg)
record_dir_config_file_path = path / record_dir_config_file_name
if not record_dir_config_file_path.exists():
@ -494,7 +507,7 @@ def create_endpoints_for_collection(
(
'validate',
'validate/record',
_endpoint_template,
_endpoint_validate_template,
'validate_record',
'validate',
f'Validate records for collection "{collection_name}"',
@ -556,8 +569,6 @@ def delete_endpoint(
operation_path: str,
app: FastAPI,
):
from fastapi.routing import _IncludedRouter
remove_paths_set = {
f'/{collection_name}/{operation_path}/{class_name}'
for class_name in active_classes
@ -579,8 +590,9 @@ def store_record(
class_name: str,
model: Any,
input_format: Format,
add_submission_tag: bool,
api_key: str | None = Depends(api_key_header_scheme),
*,
add_submission_tag: bool,
) -> JSONResponse | PlainTextResponse:
if input_format == Format.json and isinstance(data, str):
raise HTTPException(
@ -615,7 +627,6 @@ def store_record(
)
final_permissions = join_default_token_permissions(
abstract_config,
instance_state,
token_permissions,
collection,
)

6
dump_things_service/collection_endpoints.py
View file

@ -95,6 +95,7 @@ async def replace_collection(
async def create_or_replace_collection(
body: CollectionRequest,
api_key: str,
*,
allow_replace: bool,
):
instance_state = get_instance_state()
@ -266,8 +267,9 @@ def validate_incoming_paths(
token_collection_info = token_info.collections.get(collection_request.name)
if token_collection_info:
token_permissions = get_token_permissions(token_collection_info.mode)
if token_permissions.incoming_write or token_permissions.zones_access:
if not collection_request.incoming:
if (
token_permissions.incoming_write or token_permissions.zones_access
) and not collection_request.incoming:
detail = (
f"Cannot add collection '{collection_request.name}' without "
f"`incoming` path, because at least token '{token_name}' "

6
dump_things_service/commands/check_pids.py
View file

@ -11,8 +11,6 @@ from dump_things_service.abstract_config import (
get_config_labels,
read_config,
)
from dump_things_service.backends.schema_type_layer import _SchemaTypeLayer
from dump_things_service.backends.sqlite import _SQLiteBackend
from dump_things_service.exceptions import CurieResolutionError
from dump_things_service.instance_state import create_instance_state
from dump_things_service.manifest import manifest_configuration
@ -40,7 +38,7 @@ parser.add_argument(
def check_pids_in_stores(stores: Iterable[_ModelStore]) -> int:
result = 0
for store in stores:
print('checking', store.get_uri(), file=sys.stderr)
print('checking', store.get_uri(), file=sys.stderr) # noqa T201 -- cli result output
for record_info in store.get_all_objects():
pid = record_info.json_object['pid']
try:
@ -97,7 +95,7 @@ def main():
arguments = parser.parse_args()
result = check_pids(Path(arguments.store).absolute())
if result > 0:
print(f'found {result} unresolvable pids', file=sys.stderr)
print(f'found {result} unresolvable pids', file=sys.stderr) # noqa T201 -- cli result output
return 1
return 0

2
dump_things_service/commands/create_merged_schema.py
View file

@ -64,7 +64,7 @@ def main():
allow_unicode=True,
sort_keys=False,
)
print(text)
print(text) # noqa T201 -- cli result output
return 0

11
dump_things_service/commands/download_config.py
View file

@ -7,6 +7,7 @@ from argparse import ArgumentParser
import requests
import yaml
from starlette.status import HTTP_300_MULTIPLE_CHOICES
parser = ArgumentParser(
prog='Download a complete configuration of a running service',
@ -59,7 +60,7 @@ def main():
admin_token = os.environ.get('DTS_ADMIN_TOKEN')
if not admin_token:
print(
print( # noqa T201 -- cli result output
'An admin token must be provided in the environment variable `DTS_ADMIN_TOKEN`',
file=sys.stderr,
flush=True,
@ -73,9 +74,9 @@ def main():
)
if arguments.format == 'json':
print(json.dumps(configuration, indent=2, sort_keys=False))
print(json.dumps(configuration, indent=2, sort_keys=False)) # noqa T201 -- cli result output
elif arguments.format == 'yaml':
print(
print( # noqa T201 -- cli result output
yaml.dump(
data=configuration,
sort_keys=False,
@ -169,8 +170,8 @@ def _get_data(
token: str,
content_class: str,
) -> list:
result = requests.get(url, headers={'x-dumpthings-token': token})
if result.status_code >= 300:
result = requests.get(url, headers={'x-dumpthings-token': token}, timeout=10)
if result.status_code >= HTTP_300_MULTIPLE_CHOICES:
msg = f'Error downloading {content_class} from {url}: {result.text}'
raise RuntimeError(msg)
return result.json()

2
dump_things_service/commands/gitaudit_rebuild_index.py
View file

@ -19,7 +19,7 @@ def main():
arguments = parser.parse_args()
audit_backend = GitAuditBackend(Path(arguments.audit_store))
audit_backend._rebuild_index()
audit_backend.rebuild_index()
return 0

4
dump_things_service/commands/gitaudit_report.py
View file

@ -32,7 +32,7 @@ def main():
try:
re.compile(arguments.pid)
except re.error as e:
print('Error in PID pattern:', e, file=sys.stderr, flush=True)
print('Error in PID pattern:', e, file=sys.stderr, flush=True) # noqa T201 -- cli result output
return 1
audit_backend = GitAuditBackend(Path(arguments.audit_store))
@ -47,7 +47,7 @@ def main():
'diff': change[2],
'resulting-record': change[3],
}
print(json.dumps(report, ensure_ascii=False), flush=True)
print(json.dumps(report, ensure_ascii=False), flush=True) # noqa T201 -- cli result output
return 0

4
dump_things_service/commands/hash_token.py
View file

@ -23,10 +23,10 @@ def main():
token = arguments.token.strip()
if any(s.isspace() for s in token):
print('Whitespace are not allowed in token', file=sys.stderr, flush=True)
print('Whitespace are not allowed in token', file=sys.stderr, flush=True) # noqa T201 -- cli result output
return 1
print(hash_token_representation(token))
print(hash_token_representation(token)) # noqa T201 -- cli result output
return 0

32
dump_things_service/commands/upload_config.py
View file

@ -9,9 +9,12 @@ from pathlib import Path
import requests
import yaml
from starlette.status import HTTP_300_MULTIPLE_CHOICES
from dump_things_service.instance_state import get_record_dir_config
CONFIG_VERSION = 2
parser = ArgumentParser(
prog='Establish a configuration in a running service',
description='Read a configuration from a dump-things configuration-file '
@ -76,7 +79,7 @@ def main():
elif file_type == 'yaml':
configuration = yaml.safe_load(config_file)
else:
print(
print( # noqa T201 -- cli result output
f'Unknown file type {config_file_path} (use `.json` or `.yaml` suffix, or specify the format with --format <json|yaml>)',
file=sys.stderr,
flush=True,
@ -86,22 +89,25 @@ def main():
if arguments.old_format:
configuration = convert_config_1_to_config_2(configuration, arguments.store)
elif arguments.store:
print(
print( # noqa T201 -- cli result output
'Warning: ignoring `--store` option because `--old-format` '
'is not provided.',
file=sys.stderr,
flush=True,
)
assert configuration['type'] == 'collections', (
'`type: collections` missing in config-file'
)
assert configuration['version'] == 2, '`version: 2` missing in config-file'
if configuration['type'] != 'collections':
msg = '`type: collections` missing in config-file'
raise ValueError(msg)
if configuration['version'] != CONFIG_VERSION:
msg = f'`version: {CONFIG_VERSION}` missing in config-file'
raise ValueError(msg)
if arguments.send_to:
admin_token = os.environ.get('DTS_ADMIN_TOKEN')
if not admin_token:
print(
print( # noqa T201 -- cli result output
'An admin token must be provided in the environment variable `DTS_ADMIN_TOKEN`',
file=sys.stderr,
flush=True,
@ -114,15 +120,16 @@ def main():
arguments.send_to.removesuffix('/'),
admin_token,
)
return 0
except RuntimeError as rte:
print(f'{rte.args[0]}', file=sys.stderr, flush=True)
print(f'{rte.args[0]}', file=sys.stderr, flush=True) # noqa T201 -- cli result output
return 2
else:
return 0
if file_type == 'json':
print(json.dumps(configuration, indent=2, sort_keys=False))
print(json.dumps(configuration, indent=2, sort_keys=False)) # noqa T201 -- cli result output
elif file_type == 'yaml':
print(
print( # noqa T201 -- cli result output
yaml.dump(
data=configuration,
sort_keys=False,
@ -271,8 +278,9 @@ def _post_data(
url,
headers={'x-dumpthings-token': token},
json=data,
timeout=20,
)
if result.status_code >= 300:
if result.status_code >= HTTP_300_MULTIPLE_CHOICES:
msg = f'Error uploading {content_class}: {content_name}: {result.text}'
raise RuntimeError(msg)

8
dump_things_service/curated.py
View file

@ -79,8 +79,8 @@ add_pagination(router)
async def read_curated_records_of_type(
collection: str,
class_name: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
):
instance_state = get_instance_state()
if class_name not in instance_state.collections[collection].active_classes:
@ -107,8 +107,8 @@ async def read_curated_records_of_type(
async def read_curated_records_of_type_paginated(
collection: str,
class_name: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]:
instance_state = get_instance_state()
if class_name not in instance_state.collections[collection].active_classes:
@ -134,8 +134,8 @@ async def read_curated_records_of_type_paginated(
)
async def read_curated_all_records(
collection: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
):
return await _read_curated_records(
collection=collection,
@ -154,8 +154,8 @@ async def read_curated_all_records(
)
async def read_curated_all_records_paginated(
collection: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]:
record_list = await _read_curated_records(
collection=collection,

8
dump_things_service/incoming.py
View file

@ -81,8 +81,8 @@ async def incoming_read_records_of_type(
collection: str,
label: str,
class_name: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
):
instance_state = get_instance_state()
if class_name not in instance_state.collections[collection].active_classes:
@ -111,8 +111,8 @@ async def incoming_read_records_of_type_paginated(
collection: str,
label: str,
class_name: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]:
instance_state = get_instance_state()
if class_name not in instance_state.collections[collection].active_classes:
@ -140,8 +140,8 @@ async def incoming_read_records_of_type_paginated(
async def incoming_read_all_records(
collection: str,
label: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
):
return await _incoming_read_records(
collection=collection,
@ -162,8 +162,8 @@ async def incoming_read_all_records(
async def incoming_read_all_records_paginated(
collection: str,
label: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
api_key: str | None = Depends(api_key_header_scheme),
) -> Page[dict]:
record_list = await _incoming_read_records(
collection=collection,

4
dump_things_service/instance_state.py
View file

@ -109,7 +109,7 @@ def create_instance_state(
bootstrap_token: str,
fastapi_app: FastAPI,
) -> InstanceState:
global g_instance_state
global g_instance_state # noqa PLW0603 -- this is cached on the first call
if g_instance_state:
logger.warning('create_instance_state() already called')
@ -123,8 +123,6 @@ def create_instance_state(
def get_instance_state() -> InstanceState:
global g_instance_state
if not g_instance_state:
msg = 'get_instance_state() called before create_instance_state()'
raise RuntimeError(msg)

4
dump_things_service/lazy_list.py
View file

@ -24,7 +24,7 @@ from abc import (
ABCMeta,
abstractmethod,
)
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, SupportsIndex
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@ -58,7 +58,7 @@ class LazyList(list, metaclass=ABCMeta):
def __len__(self) -> int:
return len(self.list_info)
def __getitem__(self, index: int) -> Any:
def __getitem__(self, index: SupportsIndex) -> Any:
if isinstance(index, slice):
start = 0 if index.start is None else index.start
stop = len(self) if index.stop is None else index.stop

31
dump_things_service/main.py
View file

@ -49,6 +49,8 @@ from dump_things_service.abstract_config import (
check_collection,
hash_token_representation,
read_config,
CONFIG_VERSION_1_ID,
CONFIG_VERSION_2_ID,
)
from dump_things_service.api_key import api_key_header_scheme
from dump_things_service.converter import (
@ -184,7 +186,7 @@ if not arguments.admin_token_hash:
)
# Validate the hash token format
elif not hash_matcher.match(arguments.admin_token_hash):
print(
print( # noqa T201 -- cli result output
'Hashed admin token is not a 64-digits hex-number',
file=sys.stderr,
flush=True,
@ -204,8 +206,8 @@ else:
store_path = Path(arguments.store).resolve()
if not store_path.exists():
logger.error(f'Store path does not exist: {store_path}')
raise SystemExit(1)
logger.error('Store path does not exist: %s', str(store_path))
sys.exit(1)
disable_installed_extensions_check()
@ -253,7 +255,7 @@ def initialize_from_config_file(
config_dict = yaml.safe_load(f)
config_version = config_dict['version']
if config_version == 1:
if config_version == CONFIG_VERSION_1_ID:
logger.info(
'Converting version 1 configuration at %s',
arguments.config,
@ -262,7 +264,7 @@ def initialize_from_config_file(
config_dict,
instance_state.store_path,
)
elif config_version != 2:
elif config_version != CONFIG_VERSION_2_ID:
msg = f'Invalid version in config file: {config_version}'
raise ValueError(msg)
@ -305,12 +307,15 @@ if not (
# If there are no structures in the configuration, check for a bootstrap token.
if not (
if (
not (
g_configuration.admin_tokens
or g_configuration.collections
or g_configuration.tokens
) and not g_instance_state.bootstrap_token:
print(
)
and not g_instance_state.bootstrap_token
):
print( # noqa T201 -- cli result output
'The server has an empty configuration and requires a bootstrap '
'token (use `--admin-token-hash` to provide one)',
file=sys.stderr,
@ -400,8 +405,8 @@ async def maintenance(
async def read_record_with_pid(
collection: str,
pid: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme),
):
check_collection(g_configuration, collection)
@ -442,9 +447,9 @@ async def read_record_with_pid(
)
async def read_all_records(
collection: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme),
):
return await _read_all_records(
collection=collection,
@ -465,9 +470,9 @@ async def read_all_records(
)
async def read_all_records_paginated(
collection: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme),
) -> Page[dict | str]:
result_list = await _read_all_records(
collection=collection,
@ -487,9 +492,9 @@ async def read_all_records_paginated(
async def read_records_of_type(
collection: str,
class_name: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme),
):
return await _read_records_of_type(
collection=collection,
@ -512,9 +517,9 @@ async def read_records_of_type(
async def read_records_of_type_paginated(
collection: str,
class_name: str,
api_key: Annotated[str, Depends(api_key_header_scheme)],
matching: str | None = None,
format: Format = Format.json, # noqa: A002
api_key: str = Depends(api_key_header_scheme),
) -> Page[dict | str]:
result_list = await _read_records_of_type(
collection=collection,

9
dump_things_service/model.py
View file

@ -73,7 +73,7 @@ def get_subclasses_2(
collection_name: str,
class_name: str,
) -> list[str]:
from dump_things_service.instance_state import get_instance_state
from dump_things_service.instance_state import get_instance_state # noqa PLC0415 -- global import leads to circular imports
instance_state = get_instance_state()
schema_view = instance_state.schema_info[collection_name].schema_view
@ -84,7 +84,7 @@ def compile_module_with_increasing_recursion_limit(
pydantic_generator: PydanticGenerator,
schema_location: str,
) -> ModuleType:
global current_recursion_limit
global current_recursion_limit # noqa PLW0603 -- global state is updated from within the call-tree
module = None
module_name = (
@ -107,8 +107,9 @@ def compile_module_with_increasing_recursion_limit(
sys.setrecursionlimit(current_recursion_limit)
lgr.warning(
'RecursionError when building Pydantic model for schema '
f'{schema_location}, increasing recursion limit to: '
f'{current_recursion_limit}.'
'%s, increasing recursion limit to: %d',
schema_location,
current_recursion_limit,
)
return module

7
dump_things_service/store/model_store.py
View file

@ -1,6 +1,9 @@
from __future__ import annotations
from datetime import datetime
from datetime import (
UTC,
datetime,
)
from itertools import chain
from typing import TYPE_CHECKING
@ -100,7 +103,7 @@ class _ModelStore:
submitter_curie_or_iri = self.get_curie(self.tags['id'])
time_curie_or_iri = self.get_curie(self.tags['time'])
json_object['annotations'][submitter_curie_or_iri] = submitter
json_object['annotations'][time_curie_or_iri] = datetime.now().isoformat()
json_object['annotations'][time_curie_or_iri] = datetime.now(tz=UTC).isoformat()
@staticmethod
def homogenize_annotations(json_object) -> dict:

5
dump_things_service/tests/fixtures.py
View file

@ -7,6 +7,7 @@ from types import ModuleType
import pytest
import yaml
from fastapi.testclient import TestClient
from dump_things_service.abstract_config import (
GitAuditBackendConfig,
@ -366,7 +367,7 @@ def fastapi_app_simple(dump_stores_simple):
'--ignore-default-config-file',
str(tmp_path),
]
from dump_things_service.main import app
from dump_things_service.main import app # noqa PLC0415 -- main contains one-time-code that is not guarded
sys.argv = old_sys_argv
return app, tmp_path, audit_tmp_path, admin_token
@ -374,8 +375,6 @@ def fastapi_app_simple(dump_stores_simple):
@pytest.fixture(scope='session')
def fastapi_client_simple(fastapi_app_simple):
from fastapi.testclient import TestClient
test_client = TestClient(fastapi_app_simple[0])
store_path = fastapi_app_simple[1]
audit_path = fastapi_app_simple[2]

2
dump_things_service/tests/test_basic.py
View file

@ -134,7 +134,7 @@ def test_search_by_class(fastapi_client_simple):
)
assert response.status_code == HTTP_200_OK
json_result = response.json()
if len(json_result) == 3: # noqa: PLR2004
if len(json_result) == 3:
assert response.json() == [
{
'given_name': 'curated',

9
dump_things_service/tests/test_curated.py
View file

@ -40,7 +40,7 @@ def test_read_curated_records(
else:
assert len(json_object) == 3
for pattern, count in (('%25wolf%25', 1), ('%25cura%25', 2)):
for pattern, occurrences in (('%25wolf%25', 1), ('%25cura%25', 2)):
test_client, _, _ = fastapi_client_simple
response = test_client.get(
f'/collection_8/curated/records/{paginate}{class_name}?matching={pattern}',
@ -49,9 +49,9 @@ def test_read_curated_records(
assert response.status_code == HTTP_200_OK
json_object = response.json()
if 'items' in json_object:
assert len(json_object['items']) == count
assert len(json_object['items']) == occurrences
else:
assert len(json_object) == count
assert len(json_object) == occurrences
def test_read_curated_records_by_pid(fastapi_client_simple):
@ -179,8 +179,7 @@ def test_audit_backend_auto_flush(fastapi_client_simple):
for i in count():
if not audit_backend.current_change_set:
break
i += 1
if i == 10:
if (i + 1) == 10:
msg = 'auto flush did not trigger within 10 seconds'
raise ValueError(msg)
time.sleep(1)

2
dump_things_service/tests/test_incoming.py
View file

@ -32,7 +32,7 @@ zones_filled = False
def fill_zones(test_client):
global zones_filled
global zones_filled # noqa PLW0603 -- records global state from within test harness calls
if zones_filled:
return

6
dump_things_service/token_endpoints.py
View file

@ -1,6 +1,6 @@
import logging
import random
import re
import secrets
from typing import Annotated
from urllib.parse import quote
@ -58,7 +58,7 @@ class AdminTokenRequest(AdminTokenConfig):
def get_token_parts(token: str) -> list[str]:
parts = token.split('-', 1)
if len(parts) != 2:
if len(parts) != 2: # noqa PLR2004 -- explicit check for two parts
msg = 'Invalid token format'
raise ValueError(msg)
return parts
@ -156,7 +156,7 @@ def create_or_replace_token(
# Generate a random representation that does not yet exist.
collision = True
while collision:
body.representation = random.randbytes(24).hex()
body.representation = secrets.token_bytes(24).hex()
existing_token_info = get_token_info_by_representation(
abstract_config=abstract_config,
token_representation=body.representation,

23
dump_things_service/utils.py
View file

@ -110,7 +110,6 @@ def wrap_http_exception(
def join_default_token_permissions(
abstract_configuration: Configuration,
instance_state: InstanceState,
permissions: TokenPermission,
collection: str,
) -> TokenPermission:
@ -319,10 +318,10 @@ def create_token_store(
collection_name: str,
store_dir: Path,
) -> _ModelStore:
from dump_things_service.abstract_config import get_backend_and_extension
from dump_things_service.backends.schema_type_layer import SchemaTypeLayer
from dump_things_service.exceptions import ConfigError
from dump_things_service.store.model_store import ModelStore
from dump_things_service.abstract_config import get_backend_and_extension # noqa PLC0415 -- global import leads to circular imports
from dump_things_service.backends.schema_type_layer import SchemaTypeLayer # noqa PLC0415 -- global import leads to circular imports
from dump_things_service.exceptions import ConfigError # noqa PLC0415 -- global import leads to circular imports
from dump_things_service.store.model_store import ModelStore # noqa PLC0415 -- global import leads to circular imports
# One early requirement for the service was to be able to specify
# arbitrary directories for curated stores and incoming stores. This
@ -391,8 +390,8 @@ def create_record_dir_token_store_backend(
mapping_function: str,
suffix: str,
) -> _RecordDirStore:
from dump_things_service.backends.record_dir import RecordDirStore
from dump_things_service.instance_state import record_dir_config_file_name
from dump_things_service.backends.record_dir import RecordDirStore # noqa PLC0415 -- global import leads to circular imports
from dump_things_service.instance_state import record_dir_config_file_name # noqa PLC0415 -- global import leads to circular imports
# Write the configuration to the store, if it does not yet exist.
if not (store_dir / record_dir_config_file_name).exists():
@ -417,7 +416,7 @@ def write_record_dir_config(
mapping_function: str,
schema: str,
):
from dump_things_service.instance_state import record_dir_config_file_name
from dump_things_service.instance_state import record_dir_config_file_name # noqa PLC0415 -- global import leads to circular imports
record_dir_config_file_path = path / record_dir_config_file_name
if not record_dir_config_file_path.exists():
@ -436,8 +435,8 @@ def create_sqlite_token_store_backend(
store_dir: Path,
order_by: list[str],
) -> _SQLiteBackend:
from dump_things_service.backends.sqlite import SQLiteBackend
from dump_things_service.backends.sqlite import (
from dump_things_service.backends.sqlite import SQLiteBackend # noqa PLC0415 -- global import leads to circular imports
from dump_things_service.backends.sqlite import ( # noqa PLC0415 -- global import leads to circular imports
record_file_name as sqlite_record_file_name,
)
@ -466,7 +465,7 @@ async def process_token(
) -> tuple[TokenPermission, _ModelStore]:
if api_key is None:
collection_config = get_collection_config_by_name(abstract_config, collection)
token_store, token_permissions, user_id = get_token_store(
token_store, token_permissions, _user_id = get_token_store(
abstract_config,
instance_state,
collection,
@ -482,7 +481,7 @@ async def process_token(
)
final_permissions = join_default_token_permissions(
abstract_config, instance_state, token_permissions, collection
abstract_config, token_permissions, collection
)
# Check for maintenance mode

2
dump_things_service/validate.py
View file

@ -38,7 +38,6 @@ def validate_record(
class_name: str,
model: Any,
input_format: Format,
_: bool,
api_key: str | None = Depends(api_key_header_scheme),
) -> JSONResponse:
instance_state = get_instance_state()
@ -70,7 +69,6 @@ def validate_record(
)
final_permissions = join_default_token_permissions(
abstract_config,
instance_state,
token_permissions,
collection,
)

13
pyproject.toml
View file

@ -170,14 +170,27 @@ quote-style = "single"
"**/test_*" = [
# permit assert statements in tests
"S101",
# permit hard-coded passwords in tests
"S105", "S106",
# permit insecure hash in tests
"S324",
# permit standard random number generators in tests
"S311",
# permit magic values in tests
"PLR2004",
# permit relative import in tests
"TID252",
# permit versatile function names in tests
"N802",
# permit access to private members in tests
"SLF001",
]
# permit hard-coded passwords, assert, insecure hash and magic values in fixtures
"dump_things_service/tests/fixtures.py" = ["S101", "S105", "S106", "S324", "PLR2004"]
# permit relative import in subpackage root
"dump_things_service/*/__init__.py" = ["TID252"]
[tool.codespell]
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
skip = '.git*'