orinoco/dump-things-pyclient
5
0
Fork 1
Code Issues 1 Pull requests Packages Activity Actions

author_id-support #36

Merged
cmo merged 5 commits from author_id-support into master 2026-03-23 11:17:40 +00:00
Conversation 0 Commits 5 Files changed 8 +112 -11
8 changed files with 112 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

15
dump_things_pyclient/commands/dtc_plugins/auto_curate.py
View file

@ -96,6 +96,12 @@ stl_info = False
default=False, default=False,
is_flag=True, is_flag=True,
) )
@click.option(
'--author_id',
metavar='AUTHOR_ID',
default=None,
help='specify an author ID that will be stored in the audit log (AUTHOR_ID must not contain whitespace).',
)
@click.option( @click.option(
'--pid', '-p', '--pid', '-p',
metavar='PID', metavar='PID',
@ -141,6 +147,7 @@ def cli(
create_change_set, create_change_set,
post_change_set, post_change_set,
add_annotations, add_annotations,
author_id,
pid, pid,
exclude, exclude,
include, include,
@ -175,6 +182,7 @@ def cli(
create_change_set, create_change_set,
post_change_set, post_change_set,
add_annotations, add_annotations,
author_id,
pid, pid,
exclude, exclude,
include, include,
@ -200,6 +208,7 @@ def auto_curate(
create_change_set, create_change_set,
post_change_set, post_change_set,
add_annotations, add_annotations,
author_id,
pid, pid,
exclude, exclude,
include, include,
@ -361,6 +370,7 @@ def auto_curate(
destination_service_url=destination_service_url, destination_service_url=destination_service_url,
destination_collection=destination_collection, destination_collection=destination_collection,
destination_token=destination_token, destination_token=destination_token,
author_id=author_id,
pid=pid, pid=pid,
add_annotations=add_annotations, add_annotations=add_annotations,
dry_run=dry_run, dry_run=dry_run,
@ -379,6 +389,7 @@ def auto_curate(
destination_service_url=destination_service_url, destination_service_url=destination_service_url,
destination_collection=destination_collection, destination_collection=destination_collection,
destination_token=destination_token, destination_token=destination_token,
author_id=author_id,
pid=pid, pid=pid,
dry_run=dry_run, dry_run=dry_run,
session=session, session=session,
@ -403,6 +414,7 @@ def _curate_records(
destination_service_url: str, destination_service_url: str,
destination_collection: str, destination_collection: str,
destination_token: str, destination_token: str,
author_id: str | None,
pid: str | None, pid: str | None,
dry_run: bool, dry_run: bool,
session: Session, session: Session,
@ -432,6 +444,7 @@ def _curate_records(
collection=destination_collection, collection=destination_collection,
class_name=class_name, class_name=class_name,
record=record, record=record,
author_id=author_id,
token=destination_token, token=destination_token,
session=session, session=session,
) )
@ -621,6 +634,7 @@ def _post_change_set(
destination_service_url: str, destination_service_url: str,
destination_collection: str, destination_collection: str,
destination_token: str, destination_token: str,
author_id: str | None,
pid: str | None, pid: str | None,
add_annotations: bool, add_annotations: bool,
dry_run: bool, dry_run: bool,
@ -665,6 +679,7 @@ def _post_change_set(
collection=destination_collection, collection=destination_collection,
class_name=class_name, class_name=class_name,
record=record, record=record,
author_id=author_id,
token=destination_token, token=destination_token,
session=session, session=session,
) )

12
dump_things_pyclient/commands/dtc_plugins/post_records.py
View file

@ -43,6 +43,12 @@ console = Console(file=sys.stderr)
is_flag=True, is_flag=True,
help='store record directly in curated area instead of an inbox. (Note: requires a token with curator rights)' help='store record directly in curated area instead of an inbox. (Note: requires a token with curator rights)'
) )
@click.option(
'--author_id',
metavar='AUTHOR_ID',
default=None,
help='if --curated is provided, specify an author ID that will be stored in the audit log for the posted change (AUTHOR_ID must not contain whitespace).'
)
@click.option( @click.option(
'--ignore-errors', '--ignore-errors',
default=False, default=False,
@ -61,6 +67,7 @@ def cli(
collection, collection,
cls, cls,
curated, curated,
author_id,
ignore_errors, ignore_errors,
dry_run, dry_run,
): ):
@ -95,6 +102,7 @@ def cli(
collection, collection,
cls, cls,
curated, curated,
author_id,
ignore_errors, ignore_errors,
dry_run, dry_run,
) )
@ -107,6 +115,7 @@ def post_records(
collection, collection,
cls, cls,
curated, curated,
author_id,
ignore_errors, ignore_errors,
dry_run, dry_run,
) -> int: ) -> int:
@ -117,8 +126,10 @@ def post_records(
if curated: if curated:
write_record = curated_write_record write_record = curated_write_record
keyword_args = {'author_id': author_id}
else: else:
write_record = collection_write_record write_record = collection_write_record
keyword_args = {}
failed = [] failed = []
session = get_session() session = get_session()
@ -157,6 +168,7 @@ def post_records(
record=record, record=record,
token=token, token=token,
session=session, session=session,
**keyword_args,
) )
except HTTPError as e: except HTTPError as e:
_handle_http_error(failed, index, line, e) _handle_http_error(failed, index, line, e)

5
dump_things_pyclient/communicate.py
View file

@ -497,6 +497,7 @@ def curated_write_record(
collection: str, collection: str,
class_name: str, class_name: str,
record: dict, record: dict,
author_id: str | None = None,
token: str | None = None, token: str | None = None,
session: Session | None = None, session: Session | None = None,
) -> list[JSON]: ) -> list[JSON]:
@ -511,6 +512,9 @@ def curated_write_record(
:param collection: the name of the collection :param collection: the name of the collection
:param class_name: the class of the record given in `record` :param class_name: the class of the record given in `record`
:param record: dict: the record that should be written :param record: dict: the record that should be written
:param author_id: [optional] if set, send the given author ID to the
server, which will use it in audit logs. If not set, the server will
use the curator's ID as author ID.
:param token: [optional] if set, a token to authenticate against :param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint the endpoint, if None: no token will be sent to the endpoint
A given token must have curator-rights for the collection A given token must have curator-rights for the collection
@ -521,6 +525,7 @@ def curated_write_record(
return _post_to_url( return _post_to_url(
url=_build_url(service_url, collection, f'curated/record/{class_name}'), url=_build_url(service_url, collection, f'curated/record/{class_name}'),
token=token, token=token,
params={'author_id': author_id} if author_id else {},
session=session, session=session,
json=record, json=record,
) )

0
dump_things_pyclient/tests/__init__.py Normal file
View file

11
dump_things_pyclient/tests/fixtures.py
View file

@ -13,6 +13,7 @@ from dump_things_service import config_file_name
# String representation of curated- and incoming-path # String representation of curated- and incoming-path
curated = 'curated' curated = 'curated'
incoming = 'incoming' incoming = 'incoming'
audit_log = 'audit_log'
# Path to a local simple test schema # Path to a local simple test schema
schema_path = Path(__file__).parent / 'assets' / 'pyclient_testschema.yaml' schema_path = Path(__file__).parent / 'assets' / 'pyclient_testschema.yaml'
@ -36,7 +37,10 @@ collections:
submission_tags: submission_tags:
submitter_id_tag: https://submitter.example.com submitter_id_tag: https://submitter.example.com
submission_time_tag: https://time.example.com submission_time_tag: https://time.example.com
audit_backends:
- type: gitaudit
path: {{gitaudit_path}}
auto_flush_timeout: 1
tokens: tokens:
user_1: user_1:
user_id: test_user_1 user_id: test_user_1
@ -73,7 +77,10 @@ def dump_stores_simple(tmp_path_factory):
# Write the global config file, create collection directories, and write # Write the global config file, create collection directories, and write
# the collection config file. # the collection config file.
(tmp_path / config_file_name).write_text(global_config_text) global_config_with_audit = global_config_text.format(
gitaudit_path=str(tmp_path / audit_log)
)
(tmp_path / config_file_name).write_text(global_config_with_audit)
(tmp_path / curated).mkdir(parents=True, exist_ok=True) (tmp_path / curated).mkdir(parents=True, exist_ok=True)
(tmp_path / incoming).mkdir(parents=True, exist_ok=True) (tmp_path / incoming).mkdir(parents=True, exist_ok=True)
(tmp_path / curated / config_file_name).write_text(collection_1_config) (tmp_path / curated / config_file_name).write_text(collection_1_config)

26
dump_things_pyclient/tests/test_post_records.py
View file

@ -1,8 +1,11 @@
import json import json
import subprocess
import time
import yaml import yaml
from click.testing import CliRunner from click.testing import CliRunner
from .fixtures import audit_log
from dump_things_pyclient.commands.dtc import cli from dump_things_pyclient.commands.dtc import cli
from dump_things_pyclient.tests.common import read_records_from_store from dump_things_pyclient.tests.common import read_records_from_store
@ -94,7 +97,10 @@ def test_dtc_post_curated(dump_things_service):
cli, cli,
[ [
'--token=token-curator', '--token=token-curator',
'post-records', '--curated', f'http://127.0.0.1:{port}', 'collection_1', 'Person' 'post-records',
'--curated',
'--author_id', 'author_1@example.com',
f'http://127.0.0.1:{port}', 'collection_1', 'Person'
], ],
input='\n'.join( input='\n'.join(
json.dumps(record, ensure_ascii=False) for record in records json.dumps(record, ensure_ascii=False) for record in records
@ -134,6 +140,23 @@ def test_dtc_post_curated(dump_things_service):
assert len(returned_records) >= len(records) assert len(returned_records) >= len(records)
assert all(r in returned_records for r in records) assert all(r in returned_records for r in records)
# Check audit log
time.sleep(1)
audit_log_report = [
json.loads(line)
for line in subprocess.run(
['dump-things-gitaudit-report', store / audit_log, '.*' ],
capture_output = True,
check=True,
).stdout.decode().splitlines()
]
# Check that the author ID shows up in the log of `hans_*`-records
assert len(audit_log_report) >= 5, 'Audit log report: entry count mismatch'
assert all(
entry['author-id'] == 'author_1@example.com' for entry in audit_log_report
if entry['resulting-record'].find('given_name: hans_') >= 0
)
def test_dtc_post_record_any_class(dump_things_service): def test_dtc_post_record_any_class(dump_things_service):
port, store = dump_things_service port, store = dump_things_service
@ -162,6 +185,7 @@ def test_dtc_post_record_any_class(dump_things_service):
[ [
'--token=user_1', '--token=user_1',
'post-records', 'post-records',
'--author_id', 'author_1@example.org',
f'http://127.0.0.1:{port}', 'collection_1', '*', f'http://127.0.0.1:{port}', 'collection_1', '*',
], ],
input='\n'.join( input='\n'.join(

7
pyproject.toml
View file

@ -16,18 +16,19 @@ dependencies = [
[project.optional-dependencies] [project.optional-dependencies]
ttl = [ ttl = [
"dump-things-service>=5.3.0", "dump-things-service>=5.6.1",
] ]
tests = [ tests = [
"dump-things-service>=5.3.0", "dump-things-service>=5.6.1",
"pytest>=9.0.1", "pytest>=9.0.1",
] ]
[dependency-groups] [dependency-groups]
tests = [ tests = [
"datalad-core>=0.2.0",
"linkml>=1.10.0", "linkml>=1.10.0",
"linkml-runtime>=1.10.0", "linkml-runtime>=1.10.0",
"dump-things-service>=5.3.0", "dump-things-service>=5.6.1",
"pytest>=9.0.1", "pytest>=9.0.1",
] ]

47
uv.lock generated
View file

@ -363,6 +363,28 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/15/77/b2c333b9d9d908048a854943217a6dab7d2dca991cd87bea3fea6e6a4d45/curies-0.12.9-py3-none-any.whl", hash = "sha256:0f5cc8f5c72d3099dd7cf2a70a56c10664f82b52eda8072d45b7586caf3a5745", size = 70247, upload-time = "2026-01-14T12:10:14.895Z" }, { url = "https://files.pythonhosted.org/packages/15/77/b2c333b9d9d908048a854943217a6dab7d2dca991cd87bea3fea6e6a4d45/curies-0.12.9-py3-none-any.whl", hash = "sha256:0f5cc8f5c72d3099dd7cf2a70a56c10664f82b52eda8072d45b7586caf3a5745", size = 70247, upload-time = "2026-01-14T12:10:14.895Z" },
] ]
[[package]]
name = "datalad-core"
version = "0.2.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "datasalad" },
{ name = "git-annex" },
]
sdist = { url = "https://files.pythonhosted.org/packages/a3/a6/a1c7dc3fa5ea79f67f009a6f02bce58ff122477b6efbe1c1bb6adf51212f/datalad_core-0.2.0.tar.gz", hash = "sha256:5a13c76e4d07776e89842ee47ccb9ebf6d97498eaf2264500d3eb00a11f842bd", size = 88855, upload-time = "2026-03-19T17:22:36.886Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/24/f4c1e1e4f341e53459253796be7edb2bce4460b0e6f40928ef21e36b5077/datalad_core-0.2.0-py3-none-any.whl", hash = "sha256:641135090635f4b0f79493535b2b634b0975ffce07d9575d426ea69ebe519880", size = 106604, upload-time = "2026-03-19T17:22:38.241Z" },
]
[[package]]
name = "datasalad"
version = "0.6.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/04/94/5deda86a7bac80eeba4b73849a2ddef8028be5f398e326c992f24f49d850/datasalad-0.6.0.tar.gz", hash = "sha256:37fdb7fcee53ac636073d539a160a4f92b129183db3fbacfeda0ff43d4313e14", size = 59402, upload-time = "2025-07-16T09:09:14.298Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ab/93/102c724e2c364b7dfbad5e2725823a9ac4f274a13719095e5db11f17248d/datasalad-0.6.0-py3-none-any.whl", hash = "sha256:b5ee4494639019cf2bc8a2d41c5a6c44e3374c72f2be8dd6f6a3a4959e9c2389", size = 65219, upload-time = "2025-07-16T09:09:12.588Z" },
]
[[package]] [[package]]
name = "deprecated" name = "deprecated"
version = "1.3.1" version = "1.3.1"
@ -415,6 +437,7 @@ ttl = [
[package.dev-dependencies] [package.dev-dependencies]
tests = [ tests = [
{ name = "datalad-core" },
{ name = "dump-things-service" }, { name = "dump-things-service" },
{ name = "linkml" }, { name = "linkml" },
{ name = "linkml-runtime" }, { name = "linkml-runtime" },
@ -424,8 +447,8 @@ tests = [
[package.metadata] [package.metadata]
requires-dist = [ requires-dist = [
{ name = "click" }, { name = "click" },
{ name = "dump-things-service", marker = "extra == 'tests'", specifier = ">=5.3.0" }, { name = "dump-things-service", marker = "extra == 'tests'", specifier = ">=5.6.1" },
{ name = "dump-things-service", marker = "extra == 'ttl'", specifier = ">=5.3.0" }, { name = "dump-things-service", marker = "extra == 'ttl'", specifier = ">=5.6.1" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=9.0.1" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=9.0.1" },
{ name = "pyyaml" }, { name = "pyyaml" },
{ name = "requests" }, { name = "requests" },
@ -435,7 +458,8 @@ provides-extras = ["ttl", "tests"]
[package.metadata.requires-dev] [package.metadata.requires-dev]
tests = [ tests = [
{ name = "dump-things-service", specifier = ">=5.3.0" }, { name = "datalad-core", specifier = ">=0.2.0" },
{ name = "dump-things-service", specifier = ">=5.6.1" },
{ name = "linkml", specifier = ">=1.10.0" }, { name = "linkml", specifier = ">=1.10.0" },
{ name = "linkml-runtime", specifier = ">=1.10.0" }, { name = "linkml-runtime", specifier = ">=1.10.0" },
{ name = "pytest", specifier = ">=9.0.1" }, { name = "pytest", specifier = ">=9.0.1" },
@ -443,11 +467,12 @@ tests = [
[[package]] [[package]]
name = "dump-things-service" name = "dump-things-service"
version = "5.5.0" version = "5.6.1"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "aiohttp" }, { name = "aiohttp" },
{ name = "click" }, { name = "click" },
{ name = "datalad-core" },
{ name = "fastapi", extra = ["standard"] }, { name = "fastapi", extra = ["standard"] },
{ name = "fastapi-pagination" }, { name = "fastapi-pagination" },
{ name = "fsspec" }, { name = "fsspec" },
@ -460,7 +485,7 @@ dependencies = [
{ name = "uvicorn" }, { name = "uvicorn" },
] ]
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/24/c5/c9318fb6c180f38fd9d133ded13832dc6c59374c8116297b5c3e5e31663c/dump_things_service-5.5.0-py3-none-any.whl", hash = "sha256:ab96ce2777a753f433f90cfd539a4e4d6d8ef8cf3a9d26f9140487436e2a1181", size = 82716, upload-time = "2026-02-19T22:27:51.517Z" }, { url = "https://files.pythonhosted.org/packages/ba/fa/48712a02d35afbcd3403c7136f9ee986a5aac3c7e329603177ca0d4661c3/dump_things_service-5.6.1-py3-none-any.whl", hash = "sha256:653c963b38d6eb053362c52a922ec1d56dc14acfc2b9a56189d4d88bd10dd7dc", size = 90221, upload-time = "2026-03-23T08:07:46.736Z" },
] ]
[[package]] [[package]]
@ -784,6 +809,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl", hash = "sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437", size = 202505, upload-time = "2026-02-05T21:50:51.819Z" }, { url = "https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl", hash = "sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437", size = 202505, upload-time = "2026-02-05T21:50:51.819Z" },
] ]
[[package]]
name = "git-annex"
version = "10.20260316"
source = { registry = "https://pypi.org/simple" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e5/09/8a6fe029eae7c86ef36e123aa01e849245feac95d5088a34d7136fee8d9c/git_annex-10.20260316-py3-none-macosx_14_0_arm64.whl", hash = "sha256:171e0e096b7551cbedf7dc04eb5254721dbcdf9d18706fc7e926443afba777dc", size = 36279485, upload-time = "2026-03-17T17:33:31.127Z" },
{ url = "https://files.pythonhosted.org/packages/5f/a4/f69d1f801d8d5a8fc03392718f737cfe1850706ad053eb06234ddf3cfd7a/git_annex-10.20260316-py3-none-macosx_15_0_x86_64.whl", hash = "sha256:21fc91c6fb31abe38890d774ed5b39436cfd3fe589e38772be519f5e633d71b8", size = 14231272, upload-time = "2026-03-17T17:51:37.648Z" },
{ url = "https://files.pythonhosted.org/packages/25/e1/04428018a0716a95f714fc0ce1a081e56c7796273ca3a959638d5b01643a/git_annex-10.20260316-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:9ddc96a077283df17c02b287fc07874546fc9cd32dc3904d996bc841ac04cdf7", size = 26247156, upload-time = "2026-03-17T17:24:28.34Z" },
{ url = "https://files.pythonhosted.org/packages/0f/36/b5f70dade258ce23ac0a7b33e4c8184be3dadb5a9fbad0bb773927ae1248/git_annex-10.20260316-py3-none-manylinux_2_34_x86_64.whl", hash = "sha256:157ec16270aa3cd6fd45f16de8963d9f9c298c260ec0fd460c92a2a52ea5a789", size = 21888939, upload-time = "2026-03-17T17:27:02.262Z" },
{ url = "https://files.pythonhosted.org/packages/a7/12/9689ee50c88d5632e612703505b104ae41a3f2d8b9a1fd27cb2f4744c3ae/git_annex-10.20260316-py3-none-win_amd64.whl", hash = "sha256:8bc63486ee1beffd7200f4e7e3323123641dd146067b104547bd85209b00b9b6", size = 27056643, upload-time = "2026-03-17T17:31:17.57Z" },
]
[[package]] [[package]]
name = "graphviz" name = "graphviz"
version = "0.21" version = "0.21"