forked from cmo/triple-tools
23a83b2889
The dumpythingspyclient's communicate.py:get_paginated yields tuples, which this code seemingly did not expect
75 lines
2.1 KiB
Python
75 lines
2.1 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import io
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from dump_things_service.converter import Format, FormatConverter
|
|
from rdflib import Graph
|
|
|
|
from dump_things_pyclient.communicate import (
|
|
HTTPError,
|
|
get_paginated,
|
|
)
|
|
|
|
|
|
def _main():
|
|
argument_parser = argparse.ArgumentParser()
|
|
argument_parser.add_argument('schema')
|
|
argument_parser.add_argument('base_url')
|
|
argument_parser.add_argument('collection')
|
|
|
|
arguments = argument_parser.parse_args()
|
|
|
|
token = os.environ.get('DUMPTHINGS_TOKEN')
|
|
if token is None:
|
|
print('WARNING: environment variable DUMPTHINGS_TOKEN not set', file=sys.stderr, flush=True)
|
|
|
|
print(f'Creating converter for schema {arguments.schema} ...', file=sys.stderr, end='', flush=True)
|
|
converter = FormatConverter(
|
|
arguments.schema,
|
|
input_format=Format.json,
|
|
output_format=Format.ttl,
|
|
)
|
|
print(' done', file=sys.stderr, flush=True)
|
|
|
|
url_base = (
|
|
arguments.base_url
|
|
+ ('' if arguments.base_url.endswith('/') else '/')
|
|
+ arguments.collection
|
|
+ f'/records/p/'
|
|
)
|
|
|
|
g = Graph()
|
|
for json_object in get_paginated(url_base, page_size=100, token=os.environ.get('DUMPTHINGS_TOKEN')):
|
|
# the generator yields tuples, thus the index
|
|
json_obj = json_object[0]
|
|
object_class = json_obj.get('schema_type')
|
|
if object_class is None:
|
|
raise ValueError(f'No schema_type in {json_object}')
|
|
else:
|
|
class_name = re.search('([_A-Za-z0-9]*$)', object_class).group(0)
|
|
|
|
try:
|
|
ttl = converter.convert(json_obj, class_name)
|
|
except ValueError as ve:
|
|
print(f'WARNING: could not convert record {json_obj["pid"]}: {ve}', file=sys.stderr, flush=True)
|
|
continue
|
|
g.parse(io.StringIO(ttl), format='n3')
|
|
|
|
print(g.serialize(format='nt'))
|
|
return 0
|
|
|
|
|
|
def main():
|
|
try:
|
|
return _main()
|
|
except HTTPError as e:
|
|
print(f'ERROR: {e}: {e.response.text}', file=sys.stderr, flush=True)
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|