orinoco/dump-things-pyclient
5
0
Fork 1
Code Issues 1 Pull requests Packages Activity Actions

Use request sessions to reuse existing connections #25

Merged
cmo merged 13 commits from issue-24 into master 2026-02-04 21:01:58 +00:00
Conversation 0 Commits 13 Files changed 9 +225 -71
9 changed files with 225 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

35
dump_things_pyclient/commands/dtc_plugins/auto_curate.py
View file

@ -19,6 +19,7 @@ from rich.progress import track
from ...communicate import (
HTTPError,
curated_write_record,
get_session,
incoming_delete_record,
incoming_read_labels,
incoming_read_records,
@ -126,10 +127,8 @@ def cli(
dry_run,
)
except HTTPError as e:
rprint(
console.print(
f'[red]Error[/red]: {e}: {e.response.text}',
file=sys.stderr,
flush=True,
)
return 1
@ -154,6 +153,9 @@ def auto_curate(
console.print(f'[red]Error[/red]: no token was provided (use --token or DTC_TOKEN environment variable)')
return 1
if destination_collection is None:
destination_collection = collection
if destination_service_url is None:
destination_service_url = service_url
@ -173,10 +175,12 @@ def auto_curate(
if list_labels:
output = []
session = get_session()
all_labels = incoming_read_labels(
service_url=service_url,
collection=collection,
token=obj,
session=session,
)
for label in all_labels:
if include and label not in include:
@ -200,6 +204,7 @@ def auto_curate(
collection=collection,
label=label,
token=obj,
session=session,
)
# Get the first entry to find the total number of records
@ -241,19 +246,25 @@ def auto_curate(
class_name = re.search('([_A-Za-z0-9]*$)', record['schema_type']).group(0)
except (IndexError, KeyError):
global stl_info
console.print(f'[yellow]Warning[/yellow]: ignoring record with pid {record["pid"]} because `schema_type` attribute is missing.')
console.print(f'[yellow]Warning[/yellow]: ignoring record with pid [yellow]{record["pid"]}[/yellow] because `schema_type` attribute is missing.')
if not stl_info:
console.print(
' Please ensure that `schema_type` is stored in the records '
'or that the associated incoming area store has a backend with a '
'"Schema Type Layer", i.e., "record_dir+stl" or "sqlite+stl"."',
' [yellow]Please ensure that `schema_type` is stored in the records. Note: '
'if the incoming area store has a backend with a "Schema Type Layer", i.e., '
'"record_dir+stl" or "sqlite+stl", `schema_type` will not be stored on persistent '
'storage and will not be returned when retrieving records from the incoming area. '
'dump-things-service <= 5.4.0 circumvented the "Schema Type Layer", therefore they '
'will return records without `schema_type` attributes on curator access to '
'incoming areas or curated areas. Therefore it might be a good idea to NOT use a '
'"Schema Type Layer" in collections that shall be auto-curated, when using '
'dump-things-service <= 5.4.0.[/yellow]',
)
stl_info = True
continue
if dry_run:
console.print(f'WRITE record [green]"{record["pid"]}"[/green] of class "{class_name}" to "{destination_collection}@{destination_service_url}"')
console.print(f'DELETE record [green]"{record["pid"]}"[/green] from inbox "{label}" of "{collection}@{service_url}"')
console.print(f'WRITE record [green]"{record["pid"]}"[/green] of class "{class_name}" to collection "{destination_collection}" on "{destination_service_url}"')
console.print(f'DELETE record [green]"{record["pid"]}"[/green] from inbox "{label}" of collection "{collection}" on "{service_url}"')
continue
# Store record in destination collection
@ -264,12 +275,13 @@ def auto_curate(
class_name=class_name,
record=record,
token=destination_token,
session=session,
)
except HTTPError as e:
console.print(
f'[red]Error[/red]: writing record with pid {record["pid"]} failed: {e}: {e.response.text}',
)
raise
return 1
# Delete record from incoming area
try:
@ -279,12 +291,13 @@ def auto_curate(
label=label,
pid=record['pid'],
token=curator_token,
session=session,
)
except HTTPError as e:
console.print(
f'[red]ERROR[/red]: deleting record with pid {record["pid"]} failed: {e}: {e.response.text}',
)
raise
return 1
if output is not None:
rprint(json.dumps(output, ensure_ascii=False))

5
dump_things_pyclient/commands/dtc_plugins/clean_incoming.py
View file

@ -4,6 +4,7 @@ import rich_click as click
from ...communicate import (
HTTPError,
get_session,
incoming_delete_record,
incoming_read_records,
)
@ -71,11 +72,13 @@ def clean_incoming(
click.echo('ERROR: token not provided', err=True)
return 1
session = get_session()
for record, _, _, _, _ in incoming_read_records(
service_url=service_url,
collection=collection,
label=inbox_label,
token=token,
session=session,
):
if list_only:
click.echo(json.dumps(record, ensure_ascii=False))
@ -88,6 +91,6 @@ def clean_incoming(
label=inbox_label,
pid=record['pid'],
token=token,
session=session,
)
return 0

21
dump_things_pyclient/commands/dtc_plugins/delete_record.py → dump_things_pyclient/commands/dtc_plugins/delete_records.py
View file

@ -4,9 +4,12 @@ import sys
from functools import partial
import rich_click as click
from rich.progress import track
from rich.console import Console
from ...communicate import (
HTTPError,
get_session,
collection_delete_record,
curated_delete_record,
incoming_delete_record,
@ -18,6 +21,8 @@ subcommand_name = 'delete-records'
logger = logging.getLogger('delete-records')
console = Console(file=sys.stderr)
@click.command(short_help='Delete records from a dump-things collection')
@click.pass_obj
@ -83,7 +88,7 @@ def cli(
ignore_errors,
)
except HTTPError as e:
click.echo(f'ERROR: {e}: {e.response.text}', err=True)
console.print(f'[red]Error[/red]: {e}: {e.response.text}')
return 1
@ -102,16 +107,15 @@ def delete_records(
click.echo(f'WARNING: no token provided', err=True)
if incoming and curated:
click.echo(
'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
err=True,
)
console.print('[red]Error[/red]: -i/--incoming and -c/--curated are mutually exclusive')
return 1
session = get_session()
kwargs = dict(
service_url=service_url,
collection=collection,
token=token,
session=session,
)
if incoming == '-':
@ -134,17 +138,18 @@ def delete_records(
if not pids:
pids = sys.stdin
for pid in pids:
for pid in track(pids, console=console):
try:
operation(
service_url=service_url,
collection=collection,
pid=pid.strip(),
token=token,
session=session,
)
except HTTPError as e:
console.print(f'[red]Error[/red]: while deleting pid {pid}: {e}, {e.response.text}')
if ignore_errors:
click.echo(f'ERROR: while deleting pid {pid}: {e}', err=True)
continue
raise
return 1
return 0

31
dump_things_pyclient/commands/dtc_plugins/export.py
View file

@ -1,4 +1,5 @@
import json
import sys
from collections import defaultdict
from itertools import count
from pathlib import Path
@ -8,10 +9,13 @@ from typing import (
)
import rich_click as click
from rich.console import Console
from rich.progress import track
from ...communicate import (
HTTPError,
curated_read_records,
get_session,
incoming_read_labels,
incoming_read_records,
server,
@ -20,6 +24,8 @@ from ...communicate import (
subcommand_name = 'export'
console = Console(file=sys.stderr)
@click.command(short_help='Export a collection to the file system')
@click.pass_obj
@ -76,9 +82,9 @@ def cli(
ignore_errors,
)
except HTTPError as e:
click.echo(f'ERROR: {e}: {e.response.text}', err=True)
console.print(f'[red]Error[/red]: {e}: {e.response.text}')
except ValueError as e:
click.echo(f'ERROR: {e}', err=True)
console.print(f'[red]Error[/red]: {e}')
return 1
@ -92,14 +98,15 @@ def export(
token = obj
if token is None:
click.echo(f'ERROR: no token provided', err=True)
console.print(f'[red]Error[/red]: no token provided')
return 1
server_info = server(service_url)
session = get_session()
server_info = server(service_url, session=session)
collection_info = ([c for c in server_info['collections'] if c['name'] == collection] or None)[0]
if not collection_info:
click.echo(f'ERROR: no collection {collection} on service', err=True)
console.print(f'[red]Error[/red]: no collection {collection} on service')
return 1
description = {
@ -116,6 +123,7 @@ def export(
curated_destination = destination / 'curated'
curated_destination.mkdir()
console.print('Exporting records from curated area')
_store_records(
map(
lambda x: x[0],
@ -123,6 +131,7 @@ def export(
service_url=service_url,
collection=collection,
token=token,
session=session,
)
),
curated_destination,
@ -134,7 +143,9 @@ def export(
service_url=service_url,
collection=collection,
token=token,
session=session,
):
console.print(f'Exporting records from incoming area: {label}')
incoming_destination = destination / 'incoming' / label
incoming_destination.mkdir(parents=True, exist_ok=False)
_store_records(
@ -145,6 +156,7 @@ def export(
collection=collection,
label=label,
token=token,
session=session,
)
),
incoming_destination,
@ -162,16 +174,13 @@ def _store_records(
created_dirs = set()
class_counters = defaultdict(count)
for record in source:
for record in track(source, console=console):
class_name = _de_prefix(record.get('schema_type', None))
if class_name is None:
if ignore_errors:
click.echo(
f'WARNING: no `schema_type` in record `{record["pid"]}`',
err=True
)
console.print(f'[red]Error[/red]: no `schema type` in record {record["pid"]}')
continue
msg = f'no `schema_type` in record `{record["pid"]}`'
msg = f'no `schema_type` in record {record["pid"]}'
raise ValueError(msg)
next_name_for_class = f'{next(class_counters[class_name]):09d}.json'

26
dump_things_pyclient/commands/dtc_plugins/get_records.py
View file

@ -1,7 +1,9 @@
import json
import sys
from functools import partial
import rich_click as click
from rich.console import Console
from ...communicate import (
HTTPError,
@ -11,6 +13,7 @@ from ...communicate import (
curated_read_records,
curated_read_records_of_class,
curated_read_record_with_pid,
get_session,
incoming_read_labels,
incoming_read_records,
incoming_read_records_of_class,
@ -20,6 +23,8 @@ from ...communicate import (
subcommand_name = 'get-records'
console = Console(file=sys.stderr)
@click.command(short_help='Get records from a dump-things collection')
@click.pass_obj
@ -48,7 +53,7 @@ subcommand_name = 'get-records'
@click.option(
'--incoming', '-i',
metavar='LABEL',
help='read from the collection\'s inbox with label LABEL, if LABEL is "-", return labels of all collection inboxes and exit',
help='read from the collection\'s inbox with label LABEL, if LABEL is "-", print labels of all collection inboxes and exit',
)
@click.option(
'--curated', '-c',
@ -144,7 +149,7 @@ def cli(
pagination,
)
except HTTPError as e:
click.echo(f'ERROR: {e}: {e.response.text}', err=True)
console.print(f'[red]Error[/red]: {e}: {e.response.text}')
return 1
@ -167,19 +172,18 @@ def get_records(
token = obj
if token is None:
click.echo(f'WARNING: no token provided', err=True)
console.print(f'[yellow]Warning[/yellow]: no token provided')
if incoming and curated:
click.echo(
'ERROR: -i/--incoming and -c/--curated are mutually exclusive',
err=True,
)
console.print('[red]Error[/red]: -i/--incoming and -c/--curated are mutually exclusive')
return 1
session = get_session()
kwargs = dict(
service_url=service_url,
collection=collection,
token=token,
session=session,
)
if incoming == '-':
@ -187,7 +191,9 @@ def get_records(
click.echo('\n'.join(
map(
partial(json.dumps, ensure_ascii=False),
result)))
result
)
))
return 0
elif pid:
@ -251,8 +257,8 @@ def get_records(
if pagination:
for record in result:
print(json.dumps(record, ensure_ascii=False))
click.echo(json.dumps(record, ensure_ascii=False))
else:
for record in result:
print(json.dumps(record[0], ensure_ascii=False))
click.echo(json.dumps(record[0], ensure_ascii=False))
return 0

9
dump_things_pyclient/commands/dtc_plugins/maintenance.py
View file

@ -1,15 +1,20 @@
import logging
import sys
import rich_click as click
from rich.console import Console
from ...communicate import (
HTTPError,
get_session,
maintenance as communicate_maintenance,
)
logger = logging.getLogger('maintenance')
console = Console(file=sys.stderr)
subcommand_name = 'maintenance'
@ -61,13 +66,15 @@ def maintenance(
):
token = obj
if token is None:
click.echo('ERROR: no token provided', err=True)
console.print('[red]Error[/red]: no token provided')
return 1
session = get_session()
communicate_maintenance(
service_url=service_url,
collection=collection,
active=active,
token=token,
session=session,
)
return 0

4
dump_things_pyclient/commands/dtc_plugins/post_records.py
View file

@ -7,11 +7,11 @@ import rich_click as click
from rich.console import Console
from rich.progress import track
from .auto_curate import console
from ...communicate import (
HTTPError,
curated_write_record,
collection_write_record,
get_session,
)
@ -95,6 +95,7 @@ def post_records(
else:
write_record = collection_write_record
session = get_session()
for index, line in zip(count(), track(sys.stdin, console=console)):
try:
@ -112,6 +113,7 @@ def post_records(
class_name=cls,
record=record,
token=token,
session=session,
)
except HTTPError as e:
console.print(

11
dump_things_pyclient/commands/dtc_plugins/read_pages.py
View file

@ -1,16 +1,21 @@
import json
import logging
import sys
import rich_click as click
from rich.console import Console
from ...communicate import (
HTTPError,
get_paginated,
get_session,
)
logger = logging.getLogger('read-pages')
console = Console(file=sys.stderr)
@click.command(short_help='Read records from paginated dump-things endpoints')
@click.pass_obj
@ -106,8 +111,9 @@ def read_pages(
token = obj
if token is None:
click.echo(f'WARNING: no token provided', err=True)
console.print(f'[yellow]Warning[/yellow]: no token provided')
session = get_session()
result = get_paginated(
url=url,
token=token,
@ -121,7 +127,8 @@ def read_pages(
if matching is not None
else {}
),
}
},
session=session,
)
if stats:

154
dump_things_pyclient/communicate.py
View file

@ -9,6 +9,7 @@ from typing import (
)
import requests
from requests import Session
from requests.exceptions import HTTPError
from . import JSON
@ -17,6 +18,7 @@ from . import JSON
__all__ = [
'HTTPError',
'JSON',
'get_session',
'get_paginated',
'get',
'collection_get_classes',
@ -45,12 +47,21 @@ __all__ = [
logger = logging.getLogger('dump_things_pyclient')
def get_session() -> Session:
"""Return a session that can be used to reuse connections
:return: a Session-object that can be passed to most functions in this module
"""
return requests.Session()
def get_paginated(url: str,
token: str | None = None,
first_page: int = 1,
page_size: int = 100,
last_page: int | None = None,
parameters: dict[str, str] | None = None,
session: Session | None = None,
) -> Generator[tuple[JSON, int, int, int, int], None, None]:
"""Read all records from a paginated endpoint
@ -62,6 +73,7 @@ def get_paginated(url: str,
:param last_page: [optional] last page to return (default: None (return all pages))
:param parameters: [optional] parameters to pass to the endpoint, the
parameter `page` is set automatically in this function
:param session: [optional] if set it will be used for making requests
:return: a Generator yielding tuples containing the current record, the
current page number, the total number of pages, the size of the pages,
@ -72,7 +84,7 @@ def get_paginated(url: str,
return
for page in count(start=first_page):
result = _get_page(url, token, first_page=page, page_size=page_size, parameters=parameters)
result = _get_page(url, token, first_page=page, page_size=page_size, parameters=parameters, session=session)
total_pages, page_size, total_items = result['pages'], result['size'], result['total']
if total_pages == 0:
return
@ -90,6 +102,7 @@ def get_paginated(url: str,
def get(url: str,
token: str | None = None,
parameters: dict[str, str] | None = None,
session: Session | None = None,
) -> JSON:
"""Read JSON object from a non-paginated endpoint
@ -97,14 +110,16 @@ def get(url: str,
:param token: [optional] if str: token to authenticate against the endpoint,
if None: no token will be sent to the endpoint
:param parameters: [optional] parameters to pass to the endpoint
:param session: [optional] if set it will be used for making requests
:return: JSON object
"""
return _get_from_url(url, token, parameters)
return _get_from_url(url, token, parameters, session)
def collection_get_classes(service_url: str,
collection: str,
session: Session | None = None,
) -> Generator[str, None, None]:
"""Read classes that are supported by the collection
@ -114,12 +129,17 @@ def collection_get_classes(service_url: str,
:param service_url: the base URL of the service, i.e., the URL up to
`/<collection>/...` or `/server`
:param collection: the name of the collection
:param session: [optional] if set it will be used for making requests
:return: a generator yielding names of the supported classes
"""
service_url = f'{service_url[:-1]}' if service_url.endswith('/') else service_url
matcher = re.compile(f'/{collection}/record/([A-Z][_a-zA-Z0-9]*)$')
open_api_spec = _get_from_url(service_url + '/openapi.json', None)
open_api_spec = _get_from_url(
service_url + '/openapi.json',
token=None,
session=session,
)
for path in open_api_spec['paths']:
match = matcher.match(path)
if match:
@ -131,6 +151,7 @@ def collection_read_record_with_pid(service_url: str,
pid: str,
format: str = 'json',
token: str | None = None,
session: Session | None = None,
) -> dict | None:
"""Read record with the given pid from the collection on the service
@ -146,13 +167,16 @@ def collection_read_record_with_pid(service_url: str,
either `json` or `ttl`
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
:param session: [optional] if set it will be used for making requests
:return: The record, if it exists, None otherwise.
"""
return get(
url=_build_url(service_url, collection, 'record'),
token=token,
parameters={'pid': pid, 'format': format})
parameters={'pid': pid, 'format': format},
session=session,
)
def collection_read_records(service_url: str,
@ -163,6 +187,7 @@ def collection_read_records(service_url: str,
page: int = 1,
size: int = 100,
last_page: int | None = None,
session: Session | None = None,
) -> Generator[tuple[dict, int, int, int, int], None, None]:
"""Read records from the collection on the service
@ -179,6 +204,7 @@ def collection_read_records(service_url: str,
:param size: int: the number of records in an individual pages (default: 100)
:param last_page: int | None: if int, the last page that should be returned
if None, all pages following `page` will be returned
:param session: [optional] if set it will be used for making requests
:return: A generator yielding tuples containing: the current record, the
current page number, the total number of pages, the size of the
@ -192,7 +218,9 @@ def collection_read_records(service_url: str,
last_page=last_page,
parameters= {
'format': format,
**({'matching': matching} if matching else {})})
**({'matching': matching} if matching else {})},
session=session,
)
def collection_read_records_of_class(
@ -205,6 +233,7 @@ def collection_read_records_of_class(
page: int = 1,
size: int = 100,
last_page: int | None = None,
session: Session | None = None,
) -> Generator[tuple[dict, int, int, int, int], None, None]:
"""Read records of the specified class from the collection on the service
@ -222,6 +251,7 @@ def collection_read_records_of_class(
:param size: int: the number of records in an individual pages (default: 100)
:param last_page: int | None: if int, the last page that should be returned
if None, all pages following `page` will be returned
:param session: [optional] if set it will be used for making requests
:return: A generator yielding tuples containing: the current record, the
current page number, the total number of pages, the size of the
@ -235,7 +265,9 @@ def collection_read_records_of_class(
last_page=last_page,
parameters= {
'format': format,
**({'matching': matching} if matching else {})})
**({'matching': matching} if matching else {})},
session=session,
)
def collection_write_record(
@ -245,6 +277,7 @@ def collection_write_record(
record: dict | str,
format: str = 'json',
token: str | None = None,
session: Session | None = None,
) -> list[JSON]:
"""Write a record of the specified class to an inbox in the collection on the service
@ -258,6 +291,7 @@ def collection_write_record(
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
The token must have write access to incoming area in the collection
:param session: [optional] if set it will be used for making requests
:return list[JSON]: a list of records that was written. There might be more
than one record due to inlined-relations extraction. The individual
@ -268,7 +302,9 @@ def collection_write_record(
url=_build_url(service_url, collection, f'record/{class_name}'),
token=token,
params={'format': format},
**(dict(json=record) if format == 'json' else dict(data=record)))
session=session,
**(dict(json=record) if format == 'json' else dict(data=record)),
)
def collection_validate_record(
@ -278,6 +314,7 @@ def collection_validate_record(
record: dict | str,
format: str = 'json',
token: str | None = None,
session: Session | None = None,
) -> list[JSON]:
"""Validate a record of the specified class in the collection on the service
@ -294,6 +331,7 @@ def collection_validate_record(
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
The token must have write access to incoming area in the collection
:param session: [optional] if set it will be used for making requests
:return: True
"""
@ -302,7 +340,9 @@ def collection_validate_record(
url=_build_url(service_url, collection, f'validate/{class_name}'),
token=token,
params={'format': format},
**(dict(json=record) if format == 'json' else dict(data=record)))
session=session,
**(dict(json=record) if format == 'json' else dict(data=record)),
)
def collection_delete_record(
@ -310,6 +350,7 @@ def collection_delete_record(
collection: str,
pid: str,
token: str | None = None,
session: Session | None = None,
) -> bool:
"""Delete the record with the given pid from the collection on the service
@ -319,19 +360,23 @@ def collection_delete_record(
:param pid: the PID of the record that should be deleted
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
:param session: [optional] if set it will be used for making requests
:return: True if the record was deleted, False otherwise
"""
return _delete_url(
url=_build_url(service_url, collection, 'record'),
token=token,
params={'pid': pid})
params={'pid': pid},
session=session,
)
def curated_read_record_with_pid(service_url: str,
collection: str,
pid: str,
token: str | None = None,
session: Session | None = None,
) -> dict | None:
"""Read record with the given pid from curated area of the collection on the service
@ -345,13 +390,16 @@ def curated_read_record_with_pid(service_url: str,
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint. A
token must have curator-rights
:param session: [optional] if set it will be used for making requests
:return: The record, if it exists, None otherwise
"""
return get(
url=_build_url(service_url, collection, 'curated/record'),
token=token,
parameters={'pid': pid})
parameters={'pid': pid},
session=session,
)
def curated_read_records(service_url: str,
@ -361,6 +409,7 @@ def curated_read_records(service_url: str,
page: int = 1,
size: int = 100,
last_page: int | None = None,
session: Session | None = None,
) -> Generator[tuple[dict, int, int, int, int], None, None]:
"""Read records from the curated area the collection on the service
@ -379,6 +428,7 @@ def curated_read_records(service_url: str,
:param size: int: the number of records in an individual pages (default: 100)
:param last_page: int | None: if int, the last page that should be returned
if None, all pages following `page` will be returned
:param session: [optional] if set it will be used for making requests
:return: A generator yielding tuples containing: the current record, the
current page number, the total number of pages, the size of the
@ -390,7 +440,9 @@ def curated_read_records(service_url: str,
first_page=page,
page_size=size,
last_page=last_page,
parameters={'matching': matching} if matching else {})
parameters={'matching': matching} if matching else {},
session=session,
)
def curated_read_records_of_class(
@ -402,6 +454,7 @@ def curated_read_records_of_class(
page: int = 1,
size: int = 100,
last_page: int | None = None,
session: Session | None = None,
) -> Generator[tuple[dict, int, int, int, int], None, None]:
"""Read records of class `class_name` from the curated area the collection on the service
@ -421,6 +474,7 @@ def curated_read_records_of_class(
:param size: int: the number of records in an individual pages (default: 100)
:param last_page: int | None: if int, the last page that should be returned
if None, all pages following `page` will be returned
:param session: [optional] if set it will be used for making requests
:return: A generator yielding tuples containing: the current record, the
current page number, the total number of pages, the size of the
@ -432,7 +486,9 @@ def curated_read_records_of_class(
first_page=page,
page_size=size,
last_page=last_page,
parameters={'matching': matching} if matching else {})
parameters={'matching': matching} if matching else {},
session=session,
)
def curated_write_record(
@ -441,6 +497,7 @@ def curated_write_record(
class_name: str,
record: dict,
token: str | None = None,
session: Session | None = None,
) -> list[JSON]:
"""Write a record of the specified class to the curated area of the collection on the service
@ -456,13 +513,16 @@ def curated_write_record(
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
A given token must have curator-rights for the collection
:param session: [optional] if set it will be used for making requests
:return list[JSON]: a list containing the record that was written
"""
return _post_to_url(
url=_build_url(service_url, collection, f'curated/record/{class_name}'),
token=token,
json=record)
session=session,
json=record,
)
def curated_delete_record(
@ -470,6 +530,7 @@ def curated_delete_record(
collection: str,
pid: str,
token: str | None = None,
session: Session | None = None,
) -> bool:
"""Delete the record with the given pid from the curated area of the collection on the service
@ -480,17 +541,22 @@ def curated_delete_record(
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
A given token must have curator-rights for the collection
:param session: [optional] if set it will be used for making requests
:return: True if the record was deleted, False otherwise
"""
return _delete_url(
url=_build_url(service_url, collection, 'curated/record'),
token=token,
params={'pid': pid})
params={'pid': pid},
session=session,
)
def incoming_read_labels(service_url: str,
collection: str,
token: str | None = None,
session: Session | None = None,
) -> Generator[str, None, None]:
"""Read all incoming labels for the collection on the service.
@ -500,12 +566,15 @@ def incoming_read_labels(service_url: str,
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
A given token must have curator-rights for the collection
:param session: [optional] if set it will be used for making requests
:return: list[str]: a list of incoming area labels
"""
yield from _get_from_url(
url=_build_url(service_url, collection,'incoming/'),
token=token)
token=token,
session=session,
)
def incoming_read_record_with_pid(service_url: str,
@ -513,6 +582,7 @@ def incoming_read_record_with_pid(service_url: str,
label: str,
pid: str,
token: str | None = None,
session: Session | None = None,
) -> dict | None:
"""Read record with the given pid from the specified incoming area of the collection on the service
@ -527,13 +597,16 @@ def incoming_read_record_with_pid(service_url: str,
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint. A
token must have curator-rights
:param session: [optional] if set it will be used for making requests
:return: The record, if it exists, None otherwise
"""
return get(
url=_build_incoming_url(service_url, collection, label, 'record'),
token=token,
parameters={'pid': pid})
parameters={'pid': pid},
session=session,
)
def incoming_read_records(service_url: str,
@ -544,6 +617,7 @@ def incoming_read_records(service_url: str,
page: int = 1,
size: int = 100,
last_page: int | None = None,
session: Session | None = None,
) -> Generator[tuple[dict, int, int, int, int], None, None]:
"""Read records from the specified incoming area the collection on the service
@ -563,6 +637,7 @@ def incoming_read_records(service_url: str,
:param size: int: the number of records in an individual pages (default: 100)
:param last_page: int | None: if int, the last page that should be returned
if None, all pages following `page` will be returned
:param session: [optional] if set it will be used for making requests
:return: A generator yielding tuples containing: the current record, the
current page number, the total number of pages, the size of the
@ -574,7 +649,9 @@ def incoming_read_records(service_url: str,
first_page=page,
page_size=size,
last_page=last_page,
parameters={'matching': matching} if matching else {})
parameters={'matching': matching} if matching else {},
session=session,
)
def incoming_read_records_of_class(
@ -587,6 +664,7 @@ def incoming_read_records_of_class(
page: int = 1,
size: int = 100,
last_page: int | None = None,
session: Session | None = None,
) -> Generator[tuple[dict, int, int, int, int], None, None]:
"""Read records of the specified class from the specified incoming area the collection on the service
@ -607,6 +685,7 @@ def incoming_read_records_of_class(
:param size: int: the number of records in an individual pages (default: 100)
:param last_page: int | None: if int, the last page that should be returned
if None, all pages following `page` will be returned
:param session: [optional] if set it will be used for making requests
:return: A generator yielding tuples containing: the current record, the
current page number, the total number of pages, the size of the
@ -618,7 +697,9 @@ def incoming_read_records_of_class(
first_page=page,
page_size=size,
last_page=last_page,
parameters={'matching': matching} if matching else {})
parameters={'matching': matching} if matching else {},
session=session,
)
def incoming_write_record(
@ -628,6 +709,7 @@ def incoming_write_record(
class_name: str,
record: dict,
token: str | None = None,
session: Session | None = None,
) -> list[JSON]:
"""Write a record of the specified class to the specified incoming area of the collection on the service
@ -644,13 +726,16 @@ def incoming_write_record(
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
A given token must have curator-rights for the collection
:param session: [optional] if set it will be used for making requests
:return list[JSON]: a list containing the record that was written
"""
return _post_to_url(
url=_build_incoming_url(service_url, collection, label, f'record/{class_name}'),
token=token,
json=record)
json=record,
session=session,
)
def incoming_delete_record(
@ -659,6 +744,7 @@ def incoming_delete_record(
label: str,
pid: str,
token: str | None = None,
session: Session | None = None,
) -> bool:
"""Delete the record with the given pid from the specified incoming area of the collection on the service
@ -670,22 +756,27 @@ def incoming_delete_record(
:param token: [optional] if set, a token to authenticate against
the endpoint, if None: no token will be sent to the endpoint
A given token must have curator-rights for the collection
:param session: [optional] if set, it will be used for requests
:return: True if the record was deleted, False otherwise
"""
return _delete_url(
url=_build_incoming_url(service_url, collection, label,'record'),
token=token,
params={'pid': pid})
params={'pid': pid},
session=session,
)
def server(
service_url: str,
session: Session | None = None,
) -> JSON:
"""Get server-information from the service
:param service_url: the base URL of the service, i.e., the URL up to
`/<collection>/...` or `/server`
:param session: an optional requests.Session object to use for making requests
:return: information returned by the `<service_url>/server` endpoint
"""
@ -693,7 +784,8 @@ def server(
(f'{service_url[:-1]}' if service_url.endswith('/') else service_url)
+ '/server'
)
return _do_request(requests.get, url=url, token=None, params=None)
method = session.get if session else requests.get
return _do_request(method, url=url, token=None, params=None)
def maintenance(
@ -701,6 +793,7 @@ def maintenance(
collection: str,
active: bool,
token: str,
session: Session | None = None,
) -> None:
"""Activate or deactivate maintenance mode of a collection
@ -711,6 +804,7 @@ def maintenance(
non-active (`False`).
:param token: a token to authenticate against the endpoint, the token
must have curator-rights for the collection
:param session: an optional requests.Session object to use for making requests
"""
url = (
(f'{service_url[:-1]}' if service_url.endswith('/') else service_url)
@ -719,30 +813,37 @@ def maintenance(
_post_to_url(
url=url,
token=token,
json={'collection': collection, 'active': active}
session=session,
json={'collection': collection, 'active': active},
)
def _get_from_url(url: str,
token: str | None,
params: dict[str, str] | None = None,
session: Session | None = None,
) -> JSON:
return _do_request(requests.get, url, token, params=params)
method = session.get if session else requests.get
return _do_request(method, url, token, params=params)
def _post_to_url(url: str,
token: str | None,
params: dict[str, str] | None = None,
session: Session | None = None,
**kwargs,
) -> JSON:
return _do_request(requests.post, url, token, params, **kwargs)
method = session.post if session else requests.post
return _do_request(method, url, token, params=params, **kwargs)
def _delete_url(url: str,
token: str | None,
params: dict[str, str] | None = None,
session: Session | None = None,
) -> JSON:
return _do_request(requests.delete, url, token, params=params)
method = session.delete if session else requests.delete
return _do_request(method, url, token, params=params)
def _do_request(method: Callable,
@ -784,11 +885,12 @@ def _get_page(url_base: str,
first_page: int = 1,
page_size: int = 100,
parameters: dict | None = None,
session: Session | None = None,
) -> JSON:
parameters = parameters or {}
parameters['page'] = first_page
parameters['size'] = page_size
return _get_from_url(url_base, token, parameters)
return _get_from_url(url_base, token, parameters, session)
def _check_format_value(format: str) -> None: