From 9868947806a25a16e5cbe122e5019389d040ffee Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Tue, 30 Jun 2026 16:11:24 +0200 Subject: [PATCH 01/47] Commit WIP state of a user scraper (no functioning CLI yet) --- tools/scrape-juser.py | 248 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 tools/scrape-juser.py diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py new file mode 100644 index 0000000..bbb01a9 --- /dev/null +++ b/tools/scrape-juser.py @@ -0,0 +1,248 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "dump-things-pyclient @ https://hub.psychoinformatics.de/datalink/dump-things-pyclient.git", +# "pymarc", +# "rich", +# "rich-click", +# ] +# /// + +""" + +""" +import click +import logging +import uuid +from pymarc import parse_xml_to_array +from os import environ +from pathlib import Path +from urllib import parse +from requests import Session +from xml.etree import ElementTree as ET + +# global variable to store to be submitted records +to_submit = [] + + +@click.command() +@click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') +@click.option('--dtc-collection', '-c', default='public') +@click.option('--file', '-f') +def main( + file: str, + dtc_api_url: str = 'https://pool.psychoinformatics.de/api', + dtc_collection: str = 'public', +) -> None: + """ + Given an XML file with MARCXML publication data provided as --file, + parse publication records and submit them to a pool at --dtc-api-url + as Publication records, into the collection determined by + --dtc-collection. Set the environment variables JUSER_PW and JUSER_USER + to your Juser Login credentials. + """ + if environ.get('DTC_TOKEN', None) is None: + print("DTC_TOKEN required in environment! Aborting.") + return + if environ.get('JUSER_PW', None) is None or \ + environ.get('JUSER_USER', None) is None: + print("JUSER_PW and JUSER_USER required in environment! Aborting.") + return + create_records(file=file, + pool=dtc_api_url, + collection=dtc_collection) + + +if __name__ == '__main__': + main() + + +def create_records( + file: str, + pool: str, + collection: str, +) -> None: + + records = parse_xml_to_array(Path(file)) + logging.info('Found {} publication records'.format(len(records))) + # log into Juser + ses = juser_session() + # Loop over all records + for r in records: + retrieve_metadata(records, pool, ses) + # finally, submit: + submit_metadata(to_submit, pool, collection) + return + + +def submit_metadata( + records, + pool, + collection +) -> None: + # TODO: implement me! + return + + +def retrieve_metadata( + r, + pool: str, + session, +) -> dict: + # first, get the DOI and check if the publication already exists + doi = _lookup(r, '024', 'a') + if doi is None: + # When a record has no DOI, we abort. + # TODO: this could check based on other properties, e.g. title + return + # TODO: Check if a publication with this DOI is already in the pool + mrecord, do_not_edit = check_existing(doi, pool, 'XYZPublication') + # If a record already exists, check if it needs amendments. If a record + # does not exist, assemble a publication metadata record from scratch + if not mrecord: + mrecord = {'schema_type': 'xyzri:XYZPublication', + 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), + 'identifiers': [ + {'schema_type': 'dlthings:DOI', + 'notation': doi} + ] + } + + title = _lookup(r, '245', 'a') + add_or_edit_if_mutable(mrecord, 'title', title) + abstract = _lookup(r, '520', 'a') + add_or_edit_if_mutable(mrecord, 'abstract', abstract) + # TODO: This is PoF IV association. Needs a mapping to existing Topics + about = _lookup(r, '536', 'a') + add_or_edit_if_mutable(mrecord, 'about', about) + # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? + #license = _lookup(r, '915', 'a') + # Get FZJ authors + authors = _get_authors(r, pool, session) + add_or_edit_if_mutable(mrecord, 'attributed_to', authors) + to_submit.extend(mrecord) + + +def check_existing( + identifier: str, + pool: str, + XYZclass: str, +) -> (dict, list): + """Look up if a record of a given class with a given identifier (e.g., doi, + orcid) already exists in pool. + """ + # TODO: implement me! Return either the existing record as JSON or + # an empty dictionary. Use 'identifier' as reference (doi for a publication + # or ORCID for a person. + record = {} + if record: + do_not_edit = _check_for_immutable_infos(record) + else: + do_not_edit = [] + return record, do_not_edit + + +def _check_for_immutable_infos(rec: dict) -> list: + # get a list of all slots in the record. Treat those without an + # attribute about machine-generation as immutable + generated_infos = \ + [dict['value'] for dict in rec['attributes'][0]['attributes'] \ + if 'importedFrom' in rec['attributes'][0]['predicate']] + # don't touch keys if they don't have a machine-generated annotation + do_not_edit = [key for key in rec.keys() if key not in generated_infos] + return do_not_edit + + +def add_or_edit_if_mutable( + mrecord, + recordkey: str, + recordvalue: str, + do_not_edit: list +) -> dict: + if recordkey in do_not_edit: + return mrecord + if recordvalue is not None: + mrecord[recordkey] = recordvalue + return mrecord + + +def _lookup( + record, + field1: str, + field2: str +) -> str: + """Helper function to look up metadata without running into KeyErrors""" + parentfield = record.get(field1, None) + if parentfield is not None: + value = parentfield.get(field2, None) + if value is not None: + return value + return None + + +def _get_authors( + r, + pool, + juser_session +) -> list : + """Obtain the ORCIDs of authors from a publication that come from the FZJ""" + attributed_to = [] + authors = r.get_fields('700') + for author in authors: + # if the author has an FZJ affiliation, get their info + if '(DE-Juel1)' in author.subfields[1].value: + JulID = author.subfields[1].value + logging.info('Found author {} under JulID {}'.format( + author.subfields[0].value, + JulID + )) + orcid = _look_up_orcid(juser_session) + # TODO: look up orcid in person records from pool + precord = check_existing(orcid, pool, 'XZYPerson') + pid = precord.get('pid', None) + if pid is None: + # TODO: create a new person record, maybe take inspiration from + # import Wizard + # precord = ... + # TODO: collect to be created record somewhere and submit + to_submit.extend(precord) + else: + attr = {"schema_type": "dlthings:Attribution", + "object": pid} + attributed_to.extend(attr) + return attributed_to + + +def _look_up_orcid( + JulID: str, + Juser_session, +) -> str: + # URL encode the ID + ID = parse.quote_plus(JulID) + url = \ + f'https://juser.fz-juelich.de/search?ln=en&p={ID}&f=&action_search=Search&c=People&sf=author&so=d&rm=&rg=10&sc=0&of=xm' + res = Juser_session.get(url) + x = ET.fromstring(res.content.decode()) + # look up ORCID in the XML above. XML is a mess. + ORCID = \ + dict((e[0].text, e[1].text) for e in x.findall(".//*[@tag='024']"))['ORCID'] + return ORCID + + +def juser_session( + url: str = 'https://juser.fz-juelich.de/youraccount/login', +): + """Create a Session object in order to have a persisting session token + for authentication. This is required for Person searches.""" + data = {'login_method': 'FZJ eMail-Account', + 'p_un': environ.get('JUSER_USER'), + 'p_pw': environ.get('JUSER_PW'), + 'action': 'login'} + s = Session() + # required to solve challenge + s.cookies.set('APP_INIT', '1') + response = s.post(url, + data=data) + assert response.status_code == 200 + return s + -- 2.52.0 From 64c1c50e4366400ff5ba6667225ff03b8d9f75b2 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 08:45:54 +0200 Subject: [PATCH 02/47] Refactor to object-oriented style --- tools/scrape-juser.py | 408 ++++++++++++++++++++++-------------------- 1 file changed, 214 insertions(+), 194 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index bbb01a9..a536a5f 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -9,6 +9,13 @@ # /// """ +Using a local xml file with MARCXML publication data from JUSER, this script +creates and submits publication records to a given pool. + +Usage: +Download xml data with one of JUSERs generated search URLs +(see https://juser.fz-juelich.de/search_generator.py), e.g. +> curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%222026%22&sf=author&so=d&rg=&of=xm' > /tmp/2026-pubs.xml """ import click @@ -25,6 +32,203 @@ from xml.etree import ElementTree as ET to_submit = [] +class JuserScraper(object): + def __init__( + self, + pool: str, + collection: str, + xml: str, + ) -> None: + self.pool = pool + self.collection = collection + self.xml = Path(xml) + # this list stores to-be-submitted records + self.to_submit = [] + + # establish a session to Juser + self.session = self.juser_session() + + def create_records( + self + ) -> None: + + records = parse_xml_to_array(self.xml) + logging.info('Found {} publication records'.format(len(records))) + + # Loop over all records + for r in records: + self.retrieve_metadata(r) + + def submit_records( + self, + ) -> None: + # finally, submit: + # TODO IMPLEMENT ME! + return + + def retrieve_metadata( + self, + r, + ) -> None: + # first, get the DOI and check if the publication already exists + doi = self._lookup(r, '024', 'a') + if doi is None: + # When a record has no DOI, we abort. + # TODO: this could check based on other properties, e.g. title + return + # TODO: Check if a publication with this DOI is already in the pool + mrecord, do_not_edit = self.check_existing(doi, 'XYZPublication') + # If a record already exists, check if it needs amendments. If a record + # does not exist, assemble a publication metadata record from scratch + if not mrecord: + mrecord = {'schema_type': 'xyzri:XYZPublication', + 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), + 'identifiers': [ + {'schema_type': 'dlthings:DOI', + 'notation': doi}] + } + + title = self._lookup(r, '245', 'a') + self.add_or_edit_if_mutable(mrecord, 'title', title, do_not_edit) + abstract = self._lookup(r, '520', 'a') + self.add_or_edit_if_mutable(mrecord, 'abstract', abstract, do_not_edit) + # TODO: This is PoF IV association. Needs a mapping to existing Topics + about = self._lookup(r, '536', 'a') + self.add_or_edit_if_mutable(mrecord, 'about', about, do_not_edit) + # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? + # license = _lookup(r, '915', 'a') + # Get FZJ authors + authors = self._get_authors(r) + self.add_or_edit_if_mutable(mrecord, 'attributed_to', authors, do_not_edit) + self.to_submit.extend(mrecord) + + def check_existing( + self, + identifier: str, + XYZclass: str, + ) -> (dict, list): + """Look up if a record of a given class with a given identifier (e.g., doi, + orcid) already exists in pool. + """ + # TODO: implement me! Return either the existing record as JSON or + # an empty dictionary. Use 'identifier' as reference (doi for a publication + # or ORCID for a person. self.pool ... + record = {} + if record: + do_not_edit = self._check_for_immutable_infos(record) + else: + do_not_edit = [] + return record, do_not_edit + + def _check_for_immutable_infos( + self, + rec: dict, + ) -> list: + # get a list of all slots in the record. Treat those without an + # attribute about machine-generation as immutable + generated_infos = \ + [dict['value'] for dict in rec['attributes'][0]['attributes'] \ + if 'importedFrom' in rec['attributes'][0]['predicate']] + # don't touch keys if they don't have a machine-generated annotation + do_not_edit = [key for key in rec.keys() if key not in generated_infos] + return do_not_edit + + def add_or_edit_if_mutable( + self, + mrecord, + recordkey: str, + recordvalue: str, + do_not_edit: list + ) -> dict: + if recordkey in do_not_edit: + return mrecord + if recordvalue is not None: + mrecord[recordkey] = recordvalue + return mrecord + + def _lookup( + self, + record, + field1: str, + field2: str + ) -> str: + """Helper function to look up metadata without running into KeyErrors""" + parentfield = record.get(field1, None) + if parentfield is not None: + value = parentfield.get(field2, None) + if value is not None: + return value + return None + + def _get_authors( + self, + r, + ) -> list: + """Obtain the ORCIDs of authors from a publication that come from the FZJ""" + attributed_to = [] + authors = r.get_fields('700') + for author in authors: + # if the author has an FZJ affiliation, get their info + if '(DE-Juel1)' in author.subfields[1].value: + JulID = author.subfields[1].value + logging.info('Found author {} under JulID {}'.format( + author.subfields[0].value, + JulID + )) + orcid = self._look_up_orcid(JulID) + # TODO: look up orcid in person records from pool + precord = self.check_existing(orcid, 'XZYPerson') + pid = precord.get('pid', None) + if pid is None: + # TODO: create a new person record, maybe take inspiration from + # import Wizard + # precord = ... + # TODO: collect to be created record somewhere and submit + to_submit.extend(precord) + else: + attr = {"schema_type": "dlthings:Attribution", + "object": pid} + attributed_to.extend(attr) + return attributed_to + + def _look_up_orcid( + self, + JulID: str, + ) -> str: + # URL encode the ID + ID = parse.quote_plus(JulID) + url = \ + f'https://juser.fz-juelich.de/search?ln=en&p={ID}&f=&action_search=Search&c=People&sf=author&so=d&rm=&rg=10&sc=0&of=xm' + res = self.session.get(url) + x = ET.fromstring(res.content.decode()) + # look up ORCID in the XML above. XML is a mess. + try: + ORCID = \ + dict((e[0].text, e[1].text) for e in x.findall(".//*[@tag='024']"))[ + 'ORCID'] + except KeyError as e: + ORCID=None + return ORCID + + def juser_session( + self, + url: str = 'https://juser.fz-juelich.de/youraccount/login', + ): + """Create a Session object in order to have a persisting session token + for authentication. This is required for Person searches.""" + data = {'login_method': 'FZJ eMail-Account', + 'p_un': environ.get('JUSER_USER'), + 'p_pw': environ.get('JUSER_PW'), + 'action': 'login'} + s = Session() + # required to solve challenge + s.cookies.set('APP_INIT', '1') + response = s.post(url, + data=data) + assert response.status_code == 200 + return s + + @click.command() @click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') @click.option('--dtc-collection', '-c', default='public') @@ -48,201 +252,17 @@ def main( environ.get('JUSER_USER', None) is None: print("JUSER_PW and JUSER_USER required in environment! Aborting.") return - create_records(file=file, - pool=dtc_api_url, - collection=dtc_collection) + JS = JuserScraper( + pool=dtc_api_url, + collection=dtc_collection, + xml=file, + ) + JS.create_records() + # TODO: potentially do reporting about the records now + JS.submit_records() -if __name__ == '__main__': - main() +#if __name__ == '__main__': +# main() -def create_records( - file: str, - pool: str, - collection: str, -) -> None: - - records = parse_xml_to_array(Path(file)) - logging.info('Found {} publication records'.format(len(records))) - # log into Juser - ses = juser_session() - # Loop over all records - for r in records: - retrieve_metadata(records, pool, ses) - # finally, submit: - submit_metadata(to_submit, pool, collection) - return - - -def submit_metadata( - records, - pool, - collection -) -> None: - # TODO: implement me! - return - - -def retrieve_metadata( - r, - pool: str, - session, -) -> dict: - # first, get the DOI and check if the publication already exists - doi = _lookup(r, '024', 'a') - if doi is None: - # When a record has no DOI, we abort. - # TODO: this could check based on other properties, e.g. title - return - # TODO: Check if a publication with this DOI is already in the pool - mrecord, do_not_edit = check_existing(doi, pool, 'XYZPublication') - # If a record already exists, check if it needs amendments. If a record - # does not exist, assemble a publication metadata record from scratch - if not mrecord: - mrecord = {'schema_type': 'xyzri:XYZPublication', - 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), - 'identifiers': [ - {'schema_type': 'dlthings:DOI', - 'notation': doi} - ] - } - - title = _lookup(r, '245', 'a') - add_or_edit_if_mutable(mrecord, 'title', title) - abstract = _lookup(r, '520', 'a') - add_or_edit_if_mutable(mrecord, 'abstract', abstract) - # TODO: This is PoF IV association. Needs a mapping to existing Topics - about = _lookup(r, '536', 'a') - add_or_edit_if_mutable(mrecord, 'about', about) - # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? - #license = _lookup(r, '915', 'a') - # Get FZJ authors - authors = _get_authors(r, pool, session) - add_or_edit_if_mutable(mrecord, 'attributed_to', authors) - to_submit.extend(mrecord) - - -def check_existing( - identifier: str, - pool: str, - XYZclass: str, -) -> (dict, list): - """Look up if a record of a given class with a given identifier (e.g., doi, - orcid) already exists in pool. - """ - # TODO: implement me! Return either the existing record as JSON or - # an empty dictionary. Use 'identifier' as reference (doi for a publication - # or ORCID for a person. - record = {} - if record: - do_not_edit = _check_for_immutable_infos(record) - else: - do_not_edit = [] - return record, do_not_edit - - -def _check_for_immutable_infos(rec: dict) -> list: - # get a list of all slots in the record. Treat those without an - # attribute about machine-generation as immutable - generated_infos = \ - [dict['value'] for dict in rec['attributes'][0]['attributes'] \ - if 'importedFrom' in rec['attributes'][0]['predicate']] - # don't touch keys if they don't have a machine-generated annotation - do_not_edit = [key for key in rec.keys() if key not in generated_infos] - return do_not_edit - - -def add_or_edit_if_mutable( - mrecord, - recordkey: str, - recordvalue: str, - do_not_edit: list -) -> dict: - if recordkey in do_not_edit: - return mrecord - if recordvalue is not None: - mrecord[recordkey] = recordvalue - return mrecord - - -def _lookup( - record, - field1: str, - field2: str -) -> str: - """Helper function to look up metadata without running into KeyErrors""" - parentfield = record.get(field1, None) - if parentfield is not None: - value = parentfield.get(field2, None) - if value is not None: - return value - return None - - -def _get_authors( - r, - pool, - juser_session -) -> list : - """Obtain the ORCIDs of authors from a publication that come from the FZJ""" - attributed_to = [] - authors = r.get_fields('700') - for author in authors: - # if the author has an FZJ affiliation, get their info - if '(DE-Juel1)' in author.subfields[1].value: - JulID = author.subfields[1].value - logging.info('Found author {} under JulID {}'.format( - author.subfields[0].value, - JulID - )) - orcid = _look_up_orcid(juser_session) - # TODO: look up orcid in person records from pool - precord = check_existing(orcid, pool, 'XZYPerson') - pid = precord.get('pid', None) - if pid is None: - # TODO: create a new person record, maybe take inspiration from - # import Wizard - # precord = ... - # TODO: collect to be created record somewhere and submit - to_submit.extend(precord) - else: - attr = {"schema_type": "dlthings:Attribution", - "object": pid} - attributed_to.extend(attr) - return attributed_to - - -def _look_up_orcid( - JulID: str, - Juser_session, -) -> str: - # URL encode the ID - ID = parse.quote_plus(JulID) - url = \ - f'https://juser.fz-juelich.de/search?ln=en&p={ID}&f=&action_search=Search&c=People&sf=author&so=d&rm=&rg=10&sc=0&of=xm' - res = Juser_session.get(url) - x = ET.fromstring(res.content.decode()) - # look up ORCID in the XML above. XML is a mess. - ORCID = \ - dict((e[0].text, e[1].text) for e in x.findall(".//*[@tag='024']"))['ORCID'] - return ORCID - - -def juser_session( - url: str = 'https://juser.fz-juelich.de/youraccount/login', -): - """Create a Session object in order to have a persisting session token - for authentication. This is required for Person searches.""" - data = {'login_method': 'FZJ eMail-Account', - 'p_un': environ.get('JUSER_USER'), - 'p_pw': environ.get('JUSER_PW'), - 'action': 'login'} - s = Session() - # required to solve challenge - s.cookies.set('APP_INIT', '1') - response = s.post(url, - data=data) - assert response.status_code == 200 - return s - -- 2.52.0 From b39521b23cd6337800e5f3d5a7721bda9c78b2b6 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 20:24:21 +0200 Subject: [PATCH 03/47] towards Publication records --- tools/scrape-juser.py | 340 ++++++++++++++++++++++++++++-------------- 1 file changed, 230 insertions(+), 110 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index a536a5f..cdc91ea 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -19,8 +19,10 @@ Download xml data with one of JUSERs generated search URLs """ import click +import json import logging import uuid +from collections.abc import Callable from pymarc import parse_xml_to_array from os import environ from pathlib import Path @@ -28,8 +30,94 @@ from urllib import parse from requests import Session from xml.etree import ElementTree as ET -# global variable to store to be submitted records -to_submit = [] + +def juser_session( + url: str = 'https://juser.fz-juelich.de/youraccount/login', +): + """Create a Session object in order to have a persisting session token + for authentication. This is required for Person searches.""" + data = {'login_method': 'FZJ eMail-Account', + 'p_un': environ.get('JUSER_USER'), + 'p_pw': environ.get('JUSER_PW'), + 'action': 'login'} + s = Session() + # required to solve challenge + s.cookies.set('APP_INIT', '1') + response = s.post(url, + data=data) + assert response.status_code == 200 + return s + + +def _check_for_immutable_infos( + rec: dict, +) -> list: + # get a list of all slots in the record. Treat those without an + # attribute about machine-generation as immutable + if rec.get('attributes', None) is None: + # no attributes present -- all keys are human annotated! + do_not_edit = [key for key in rec.keys()] + else: + generated_infos = \ + [dict['value'] for dict in rec['attributes'][0]['attributes'] \ + if 'importedFrom' in rec['attributes'][0]['predicate']] + # don't touch keys if they don't have a machine-generated annotation + do_not_edit = [key for key in rec.keys() if key not in generated_infos] + return rec, do_not_edit + + +def add_or_edit_if_mutable( + mrecord, + recordkey: str, + recordvalue: str, + do_not_edit: list +) -> dict: + if recordkey in do_not_edit: + return mrecord + if recordvalue is not None: + mrecord[recordkey] = recordvalue + return mrecord + + +# TODO: from scrape-calendar.py -- factor out and import! +def add_machine_prov( + generated_infos, + juser_id, + rec, + scriptpid='xyzrins:instruments/d3126ff5-623c-48af-ac48-4d921ef9b80d' +): + if generated_infos is None: + return rec + url = 'https://juser.fz-juelich.de/record/' + juser_id + prov = {'attributes': [ + {'predicate': 'http://purl.org/pav/importedFrom', + 'value': url, + 'attributes': [] + }]} + for value in generated_infos: + new = {'predicate': 'prov:generated', + 'value': value, + 'characterized_by': [{ + 'predicate': 'prov:generated_by', + 'object': scriptpid + }]} + prov['attributes'][0]['attributes'].append(new) + rec.update(prov) + return rec + + +def _lookup( + record, + field1: str, + field2: str +) -> str: + """Helper function to look up metadata without running into KeyErrors""" + parentfield = record.get(field1, None) + if parentfield is not None: + value = parentfield.get(field2, None) + if value is not None: + return value + return None class JuserScraper(object): @@ -38,15 +126,21 @@ class JuserScraper(object): pool: str, collection: str, xml: str, + persons, + pubs ) -> None: self.pool = pool self.collection = collection self.xml = Path(xml) + # caches of dumpthings records + self.persons = map_record_to_feature(process_orcid, persons) + self.pubs = map_record_to_feature(process_doi, pubs) + # cache of JulID to ORCID associations + self.JulIDs = {} # this list stores to-be-submitted records self.to_submit = [] - # establish a session to Juser - self.session = self.juser_session() + self.session = juser_session() def create_records( self @@ -64,6 +158,8 @@ class JuserScraper(object): ) -> None: # finally, submit: # TODO IMPLEMENT ME! + from pprint import pprint + pprint(self.to_submit) return def retrieve_metadata( @@ -71,100 +167,78 @@ class JuserScraper(object): r, ) -> None: # first, get the DOI and check if the publication already exists - doi = self._lookup(r, '024', 'a') + doi = _lookup(r, '024', 'a') if doi is None: # When a record has no DOI, we abort. # TODO: this could check based on other properties, e.g. title return - # TODO: Check if a publication with this DOI is already in the pool - mrecord, do_not_edit = self.check_existing(doi, 'XYZPublication') + # Check if a publication with this DOI is already in the pool # If a record already exists, check if it needs amendments. If a record # does not exist, assemble a publication metadata record from scratch - if not mrecord: + if doi in self.pubs.keys(): + mrecord, do_not_edit = \ + _check_for_immutable_infos(self.pubs[doi]) + else: mrecord = {'schema_type': 'xyzri:XYZPublication', 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), 'identifiers': [ {'schema_type': 'dlthings:DOI', 'notation': doi}] } - - title = self._lookup(r, '245', 'a') - self.add_or_edit_if_mutable(mrecord, 'title', title, do_not_edit) - abstract = self._lookup(r, '520', 'a') - self.add_or_edit_if_mutable(mrecord, 'abstract', abstract, do_not_edit) - # TODO: This is PoF IV association. Needs a mapping to existing Topics - about = self._lookup(r, '536', 'a') - self.add_or_edit_if_mutable(mrecord, 'about', about, do_not_edit) - # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? - # license = _lookup(r, '915', 'a') - # Get FZJ authors - authors = self._get_authors(r) - self.add_or_edit_if_mutable(mrecord, 'attributed_to', authors, do_not_edit) - self.to_submit.extend(mrecord) - - def check_existing( - self, - identifier: str, - XYZclass: str, - ) -> (dict, list): - """Look up if a record of a given class with a given identifier (e.g., doi, - orcid) already exists in pool. - """ - # TODO: implement me! Return either the existing record as JSON or - # an empty dictionary. Use 'identifier' as reference (doi for a publication - # or ORCID for a person. self.pool ... - record = {} - if record: - do_not_edit = self._check_for_immutable_infos(record) - else: do_not_edit = [] - return record, do_not_edit - - def _check_for_immutable_infos( - self, - rec: dict, - ) -> list: - # get a list of all slots in the record. Treat those without an - # attribute about machine-generation as immutable - generated_infos = \ - [dict['value'] for dict in rec['attributes'][0]['attributes'] \ - if 'importedFrom' in rec['attributes'][0]['predicate']] - # don't touch keys if they don't have a machine-generated annotation - do_not_edit = [key for key in rec.keys() if key not in generated_infos] - return do_not_edit - - def add_or_edit_if_mutable( - self, - mrecord, - recordkey: str, - recordvalue: str, - do_not_edit: list - ) -> dict: - if recordkey in do_not_edit: - return mrecord - if recordvalue is not None: - mrecord[recordkey] = recordvalue - return mrecord - - def _lookup( - self, - record, - field1: str, - field2: str - ) -> str: - """Helper function to look up metadata without running into KeyErrors""" - parentfield = record.get(field1, None) - if parentfield is not None: - value = parentfield.get(field2, None) - if value is not None: - return value - return None + generated_infos = [] + # record of the juser ID, for machine provenance and linking to the + # original + juser_id = r.fields[0].data + if 'title' not in do_not_edit: + title = _lookup(r, '245', 'a') + mrecord['title'] = title + generated_infos.append('title') + if 'description' not in do_not_edit: + abstract = _lookup(r, '520', 'a') + mrecord['description'] = abstract + generated_infos.append('description') + if 'attributed_to' not in do_not_edit: + # add authors. The function returns a list of dicts + authors = self._get_authors(r) + if len(authors) > 0: + if "attributed_to" not in mrecord.keys(): + mrecord["attributed_to"] = authors + else: + for author_dict in authors: + # check if the author is already part of the record + if any( + [author_dict['object'] in attr['object'] + for attr in mrecord['attributed_to']] + ): + continue + else: + mrecord["attributed_to"].append(author_dict) + generated_infos.append('attributed_to') + if 'about' not in do_not_edit: + # about is a list + about = _lookup(r, '536', 'a') + # TODO: This is PoF IV association. Needs a mapping to existing Topics + # generated_infos.append('about') + if 'rules' not in do_not_edit: + # rules is a list + # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? + license = _lookup(r, '915', 'a') + # generated_infos.append('rules') + # add machine-prov + mrecord = add_machine_prov( + generated_infos, + juser_id, + mrecord + ) + self.to_submit.append(mrecord) def _get_authors( self, r, - ) -> list: - """Obtain the ORCIDs of authors from a publication that come from the FZJ""" + ) -> list[dict]: + """Obtain the ORCIDs of authors from a publication that + come from the FZJ""" attributed_to = [] authors = r.get_fields('700') for author in authors: @@ -176,25 +250,35 @@ class JuserScraper(object): JulID )) orcid = self._look_up_orcid(JulID) - # TODO: look up orcid in person records from pool - precord = self.check_existing(orcid, 'XZYPerson') - pid = precord.get('pid', None) + pid = self.persons[orcid] if orcid is not None and orcid in self.persons.keys() \ + else None + else: + # if there is no ORCID, we give up + continue if pid is None: # TODO: create a new person record, maybe take inspiration from # import Wizard - # precord = ... - # TODO: collect to be created record somewhere and submit - to_submit.extend(precord) + p_pid = 'xyzrins:persons/' + str(uuid.uuid4()), + p_rec = {'pid': p_pid, + 'given_name': 'TODO', + 'family_name': 'TODO'} # TODO this needs name parsing of sorts... + # collect to be created record for submission + #self.to_submit.append(p_rec) + attr = {"schema_type": "dlthings:Attribution", + "object": p_pid} else: attr = {"schema_type": "dlthings:Attribution", "object": pid} - attributed_to.extend(attr) + attributed_to.append(attr) return attributed_to def _look_up_orcid( self, JulID: str, ) -> str: + if JulID in self.JulIDs.keys(): + # this ID is already known, don't query again + return self.JulIDs[JulID] # URL encode the ID ID = parse.quote_plus(JulID) url = \ @@ -206,37 +290,69 @@ class JuserScraper(object): ORCID = \ dict((e[0].text, e[1].text) for e in x.findall(".//*[@tag='024']"))[ 'ORCID'] + # add to cache + self.JulIDs[JulID] = ORCID except KeyError as e: - ORCID=None + ORCID = None return ORCID - def juser_session( - self, - url: str = 'https://juser.fz-juelich.de/youraccount/login', - ): - """Create a Session object in order to have a persisting session token - for authentication. This is required for Person searches.""" - data = {'login_method': 'FZJ eMail-Account', - 'p_un': environ.get('JUSER_USER'), - 'p_pw': environ.get('JUSER_PW'), - 'action': 'login'} - s = Session() - # required to solve challenge - s.cookies.set('APP_INIT', '1') - response = s.post(url, - data=data) - assert response.status_code == 200 - return s + +def map_record_to_feature( + feature: Callable, + records: list[dict] +) -> dict[str, str]: + """Create a mapping of records to a specific feature (DOI or ORCIDs) + """ + my_map = {} + for record in records: + if (feat := feature(record)) is not None: + my_map[feat] = record + return my_map + + +# TODO: taken from enrich-via-doi.py -- factor out and import! +def process_doi(paper: dict) -> str | None: + """Return a DOI from identifiers""" + + for identifier in paper.get("identifiers", []): + if ( + pid_of(identifier.get("creator")) == "ror:01fyxcz70" + or identifier.get("schema_type") == "dlthings:DOI" + ): + return identifier.get("notation") + + +def pid_of(x: str | dict) -> str: + """Return a PID of an object, inlined or not + + A shortcut - makes a pid string or an inlined dict (where pid is a + property) equivalent. Does not do further validation, but it could + be added here. + + """ + return x.get("pid", "") if isinstance(x, dict) else x + + +def process_orcid(person: dict) -> str | None: + """Return an ORCID from identifiers""" + + for identifier in person.get("identifiers", []): + if pid_of(identifier.get("creator")) == "ror:04fa4r544": + return identifier.get("notation") @click.command() @click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') @click.option('--dtc-collection', '-c', default='public') @click.option('--file', '-f') +@click.option('--persons', type=click.File("rb"), default='.cache/Person.jsonl') +@click.option('--publications', type=click.File("rb"), default='.cache/Publications.jsonl') def main( file: str, - dtc_api_url: str = 'https://pool.psychoinformatics.de/api', - dtc_collection: str = 'public', + dtc_api_url: str, + dtc_collection: str, + persons, + publications, ) -> None: """ Given an XML file with MARCXML publication data provided as --file, @@ -252,10 +368,16 @@ def main( environ.get('JUSER_USER', None) is None: print("JUSER_PW and JUSER_USER required in environment! Aborting.") return + all_people = [json.loads(line) for line in + persons] if persons is not None else [] + all_pubs = [json.loads(line) for line in + publications] if persons is not None else [] JS = JuserScraper( pool=dtc_api_url, collection=dtc_collection, xml=file, + persons=all_people, + pubs=all_pubs, ) JS.create_records() # TODO: potentially do reporting about the records now @@ -264,5 +386,3 @@ def main( #if __name__ == '__main__': # main() - - -- 2.52.0 From 3900ece2476b79b7b9b0d1a4c77c7551b0f9c66a Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 20:36:53 +0200 Subject: [PATCH 04/47] add brief documentation string --- tools/scrape-juser.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index cdc91ea..e09361a 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -13,9 +13,17 @@ Using a local xml file with MARCXML publication data from JUSER, this script creates and submits publication records to a given pool. Usage: -Download xml data with one of JUSERs generated search URLs -(see https://juser.fz-juelich.de/search_generator.py), e.g. -> curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%222026%22&sf=author&so=d&rg=&of=xm' > /tmp/2026-pubs.xml +1) Download xml data with one of JUSERs generated search URLs +(see https://juser.fz-juelich.de/search_generator.py) +> mkdir .cache +> curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%222026%22&sf=author&so=d&rg=&of=xm' > .cache/juser-pubs.xml + +2) Cache Person and Publication records from your pool +> dtc get-records $DUMPTHINGS_APIURL public -C XYZPublication > .cache/Publications.jsonl +> dtc get-records $DUMPTHINGS_APIURL public -C XZYPerson > .cache/Person.jsonl + +3) Invoke the script +> uv run tools/scrape-juser.py --file .cache/juser-pubs.xml --persons /tmp/.cache/Person.jsonl --publications /tmp/.cache/Publications.jsonl """ import click -- 2.52.0 From 0a56c75b536f9ccb2ef70fce88a8ce3612a73da7 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 20:37:15 +0200 Subject: [PATCH 05/47] CI: Scrape-user workflow --- .forgejo/workflows/scrape-juser.yml | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .forgejo/workflows/scrape-juser.yml diff --git a/.forgejo/workflows/scrape-juser.yml b/.forgejo/workflows/scrape-juser.yml new file mode 100644 index 0000000..d9de6bf --- /dev/null +++ b/.forgejo/workflows/scrape-juser.yml @@ -0,0 +1,46 @@ +name: Scrape publications from JUSER + +on: + workflow_dispatch: + inputs: + year: + description: "Fetch Juser publications from this year (e.g. 2026)" + required: true + default: '2026' + type: string + +env: + DTC_TOKEN: ${{ secrets.POOLTOKEN }} + DUMPTHINGS_APIURL: https://pool.psychoinformatics.de/api + DUMPTHINGS_COLLECTION: public + JUSER_PW: ${{ secrets.JUSER_PW }} + JUSER_USER: ${{ secrets.JUSER_USER }} + +jobs: + scrape-juser: + name: Scrape JUSER + runs-on: debian-latest + defaults: + run: + shell: bash + steps: + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install metadata tools + run: | + uv tool install https://hub.psychoinformatics.de/orinoco/query-things.git \ + --with-executables-from dump-things-pyclient + - name: Fetch script + run: | + wget https://hub.psychoinformatics.de/orinoco/knowledge-enrichment/raw/branch/main/tools/scrape-juser.py + - name: Pre-fetch pool data + run: | + mkdir .cache + dtc get-records $DUMPTHINGS_APIURL public -C XYZPublication > .cache/Publications.jsonl + dtc get-records $DUMPTHINGS_APIURL public -C XZYPerson > .cache/Person.jsonl + - name: Pre-fetch Juser records + curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%22${{ inputs.year }}%22&sf=author&so=d&rg=&of=xm' > .cache/juser-pubs.xml + + - name: Process records + run: | + uv run scrape-juser.py --file .cache/juser-pubs.xml --persons .cache/Person.jsonl --publications .cache/Publications.jsonl -- 2.52.0 From 997088426ba65f8fda4a40d9db45364575de4938 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 21:44:34 +0200 Subject: [PATCH 06/47] Fix: find ORCIDs irrespective of casing --- tools/scrape-juser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index e09361a..2214479 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -296,8 +296,8 @@ class JuserScraper(object): # look up ORCID in the XML above. XML is a mess. try: ORCID = \ - dict((e[0].text, e[1].text) for e in x.findall(".//*[@tag='024']"))[ - 'ORCID'] + dict((e[0].text.casefold(), e[1].text) for e in x.findall(".//*[@tag='024']"))[ + 'orcid'] # add to cache self.JulIDs[JulID] = ORCID except KeyError as e: -- 2.52.0 From f9e9e5fb19e3ea7b7ff084143bceece4b418be7d Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 21:45:32 +0200 Subject: [PATCH 07/47] fix: remove creator conditional from orcid processing the majority of psyinf records to not have such a key in their orcid identifier dict --- tools/scrape-juser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 2214479..01b79cd 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -345,7 +345,10 @@ def process_orcid(person: dict) -> str | None: """Return an ORCID from identifiers""" for identifier in person.get("identifiers", []): - if pid_of(identifier.get("creator")) == "ror:04fa4r544": + # this is a deviation from enrich-via-doi, which filtered based on + # '("creator")) == "ror:04fa4r544"' -- in the psyinf pool, only a + # fraction of identifiers have this creator annotation + if pid_of(identifier.get("schema_type")) == "xyzri:ORCID": return identifier.get("notation") -- 2.52.0 From e673268435e75b89aeee51d5d59a8bd926abcfc4 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Wed, 1 Jul 2026 21:52:10 +0200 Subject: [PATCH 08/47] Create proper new person records --- tools/scrape-juser.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 01b79cd..400b6e6 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -250,6 +250,7 @@ class JuserScraper(object): attributed_to = [] authors = r.get_fields('700') for author in authors: + orcid = None # if the author has an FZJ affiliation, get their info if '(DE-Juel1)' in author.subfields[1].value: JulID = author.subfields[1].value @@ -258,20 +259,38 @@ class JuserScraper(object): JulID )) orcid = self._look_up_orcid(JulID) + logging.info('their orcid is {}'.format(orcid)) pid = self.persons[orcid] if orcid is not None and orcid in self.persons.keys() \ else None - else: - # if there is no ORCID, we give up + if orcid is None: + # if the author is not from Jülich or there is no ORCID, + # we don't pursue this further continue if pid is None: # TODO: create a new person record, maybe take inspiration from # import Wizard - p_pid = 'xyzrins:persons/' + str(uuid.uuid4()), + p_pid = 'xyzrins:persons/' + str(uuid.uuid4()) p_rec = {'pid': p_pid, - 'given_name': 'TODO', - 'family_name': 'TODO'} # TODO this needs name parsing of sorts... + # MARCXML does only report author names as one string + # with varying formatting (incl. or excl. initials, + # etc). Parsing this into first name and family name + # is error-prone. Instead, we use their string as a + # formatted name + 'formatted_name': author.subfields[0].value, + 'identifiers': [ + {'schema_type': 'xyzri:ORCID', + 'creator': 'ror:04fa4r544', + 'notation': orcid} + ], + 'attributes': [ + {'predicate': 'http://purl.org/pav/importedFrom', + 'value': 'xyzrins:instruments/d3126ff5-623c-48af-ac48-4d921ef9b80d'} + ] + } # collect to be created record for submission - #self.to_submit.append(p_rec) + self.to_submit.append(p_rec) + # store the author in internal cache to not resubmit + self.persons[orcid] = p_rec attr = {"schema_type": "dlthings:Attribution", "object": p_pid} else: -- 2.52.0 From 9e323f60e4092b23076f137a8a26a4c0b156976d Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 09:04:23 +0200 Subject: [PATCH 09/47] Split into protected and public submission cache --- tools/scrape-juser.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 400b6e6..b21ed9c 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -146,7 +146,8 @@ class JuserScraper(object): # cache of JulID to ORCID associations self.JulIDs = {} # this list stores to-be-submitted records - self.to_submit = [] + self.to_submit_public = [] + self.to_submit_protected = [] # establish a session to Juser self.session = juser_session() @@ -239,7 +240,8 @@ class JuserScraper(object): juser_id, mrecord ) - self.to_submit.append(mrecord) + # publication records can be public + self.to_submit_public.append(mrecord) def _get_authors( self, @@ -287,8 +289,9 @@ class JuserScraper(object): 'value': 'xyzrins:instruments/d3126ff5-623c-48af-ac48-4d921ef9b80d'} ] } - # collect to be created record for submission - self.to_submit.append(p_rec) + # collect to be created record for submission. Person records + # are protected + self.to_submit_protected.append(p_rec) # store the author in internal cache to not resubmit self.persons[orcid] = p_rec attr = {"schema_type": "dlthings:Attribution", -- 2.52.0 From bfbad024a116e9ada4062effd9d1c5b0c1fc0394 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 09:20:07 +0200 Subject: [PATCH 10/47] Save author name as display_name --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index b21ed9c..71b6fdd 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -278,7 +278,7 @@ class JuserScraper(object): # etc). Parsing this into first name and family name # is error-prone. Instead, we use their string as a # formatted name - 'formatted_name': author.subfields[0].value, + 'display_label': author.subfields[0].value, 'identifiers': [ {'schema_type': 'xyzri:ORCID', 'creator': 'ror:04fa4r544', -- 2.52.0 From 821ea6ca33a2672dae0bb205d048f501dbe675c8 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 14:19:10 +0200 Subject: [PATCH 11/47] add generated_by to publication record --- tools/scrape-juser.py | 77 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 71b6fdd..054aa36 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -63,7 +63,7 @@ def _check_for_immutable_infos( # get a list of all slots in the record. Treat those without an # attribute about machine-generation as immutable if rec.get('attributes', None) is None: - # no attributes present -- all keys are human annotated! + # no attributes present -- all existing keys are human annotated! do_not_edit = [key for key in rec.keys()] else: generated_infos = \ @@ -224,14 +224,37 @@ class JuserScraper(object): else: mrecord["attributed_to"].append(author_dict) generated_infos.append('attributed_to') - if 'about' not in do_not_edit: - # about is a list - about = _lookup(r, '536', 'a') - # TODO: This is PoF IV association. Needs a mapping to existing Topics - # generated_infos.append('about') + if 'generated_by' not in do_not_edit: + # generated_by is a list + gen = [] + # first, extract which pof project generated a publication + pof = _lookup(r, '536', 'a') + pof_pid = map_pof_to_pid.get(pof, None) + if pof_pid is not None: + pof_gen = {"schema_type": "dlthings:Generation", + "object": pof_pid} + gen.append(pof_gen) + # next, record the publication process + date = _lookup(r, '', '') + at_location = self._get_periodical(r) + if date is not None and at_location is not None: + publication_process = \ + {"at_location": "ISSN:{}".format(at_location), + "at_time": "{}".format(date), + "schema_type": "dlthings:Generation", + "object": "obo:IAO_0000444"} + gen.append(publication_process) + if 'generated_by' not in mrecord.keys(): + mrecord['generated_by'] = gen + generated_infos.append('generated_by') + else: + # TODO: figure out how to increment generated_by if some info is + # already present + pass if 'rules' not in do_not_edit: # rules is a list # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? + # TODO: see control vocab at the ZB: https://juser.fz-juelich.de/collection/HGFVOC?ln=en license = _lookup(r, '915', 'a') # generated_infos.append('rules') # add machine-prov @@ -243,6 +266,31 @@ class JuserScraper(object): # publication records can be public self.to_submit_public.append(mrecord) + + def _get_periodical( + self, + r + ) -> str: + """Obtain the ISSN of a publication venue""" + # The ZB identifies periodicals via an internal ID, e.g., + # 'PERI:(DE-600)2808093-2'We can look up the ISSN + # for an ID in their periodicals collection: + # https://juser.fz-juelich.de/collection/Periodicals?ln=en + peri = _lookup(r, '773', '0') + if peri is None: + return None + ID = parse.quote_plus(peri) + url = \ + f'https://juser.fz-juelich.de/search?ln=en&cc=Periodicals&p={ID}&f=&action_search=Search&c=Periodicals&c=&sf=&so=d&rm=&rg=10&sc=1&of=xm' + res = self.session.get(url) + x = ET.fromstring(res.content.decode()) + # look up ISSN in the XML above. XML is a mess. + try: + ISSN = x.findall(".//*[@tag='022']")[0][0].text + except IndexError as e: + ISSN = None + return ISSN + def _get_authors( self, r, @@ -269,20 +317,21 @@ class JuserScraper(object): # we don't pursue this further continue if pid is None: - # TODO: create a new person record, maybe take inspiration from - # import Wizard p_pid = 'xyzrins:persons/' + str(uuid.uuid4()) p_rec = {'pid': p_pid, # MARCXML does only report author names as one string # with varying formatting (incl. or excl. initials, # etc). Parsing this into first name and family name # is error-prone. Instead, we use their string as a - # formatted name + # display label 'display_label': author.subfields[0].value, 'identifiers': [ {'schema_type': 'xyzri:ORCID', 'creator': 'ror:04fa4r544', - 'notation': orcid} + 'notation': orcid}, + {'schema_type': 'dlthings:Identifier', + 'creator': 'https://w3id.org/isil/DE-Juel1', # ZB FZJ + 'notation': JulID} ], 'attributes': [ {'predicate': 'http://purl.org/pav/importedFrom', @@ -374,6 +423,14 @@ def process_orcid(person: dict) -> str | None: return identifier.get("notation") +map_pof_to_pid = { + 'G:(DE-HGF)POF4-5251': 'xyzrins:projects/45e542dd-2d2a-41d0-8399-90f2fde72a21', + 'G:(DE-HGF)POF4-5252': 'xyzrins:projects/858d0ddb-3c51-4cea-a0e5-a73224b59ae5', + #'G:(DE-HGF)POF4-5253': 'Neuroimaging' # Not relevant for us, not in pool + 'G:(DE-HGF)POF4-5254': 'xyzrins:projects/5340dd43-6ef0-4cf8-a936-67c2a9f3a725', + 'G:(DE-HGF)POF4-5255': 'xyzrins:projects/a3f0a9e0-c945-4e04-a698-be426a9ac075' +} + @click.command() @click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') @click.option('--dtc-collection', '-c', default='public') -- 2.52.0 From e21c9c63eb467c91cdcca9219c09614d0c6a634a Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 14:19:49 +0200 Subject: [PATCH 12/47] WIP: for testing, dump records as jsonlines --- tools/scrape-juser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 054aa36..7ee8959 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -167,8 +167,9 @@ class JuserScraper(object): ) -> None: # finally, submit: # TODO IMPLEMENT ME! - from pprint import pprint - pprint(self.to_submit) + [json.dumps(r) for r in self.to_submit_protected] + [json.dumps(r) for r in self.to_submit_public] + return def retrieve_metadata( -- 2.52.0 From b8075279afdd130da00736a120492985fb46d59f Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 14:33:32 +0200 Subject: [PATCH 13/47] fix: retrieve pid, not entire record --- tools/scrape-juser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 7ee8959..f0bbd1c 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -311,8 +311,9 @@ class JuserScraper(object): )) orcid = self._look_up_orcid(JulID) logging.info('their orcid is {}'.format(orcid)) - pid = self.persons[orcid] if orcid is not None and orcid in self.persons.keys() \ - else None + pid = self.persons[orcid]['pid'] if \ + orcid is not None and orcid in self.persons.keys() \ + else None if orcid is None: # if the author is not from Jülich or there is no ORCID, # we don't pursue this further -- 2.52.0 From 36ed29dc1e6bccd6a7cae9b39da9314b9af75c4a Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 14:40:48 +0200 Subject: [PATCH 14/47] WIP: actually print records --- tools/scrape-juser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index f0bbd1c..e4da082 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -167,8 +167,8 @@ class JuserScraper(object): ) -> None: # finally, submit: # TODO IMPLEMENT ME! - [json.dumps(r) for r in self.to_submit_protected] - [json.dumps(r) for r in self.to_submit_public] + print([json.dumps(r) for r in self.to_submit_protected]) + print([json.dumps(r) for r in self.to_submit_public]) return -- 2.52.0 From 4cd657d386b54b2fb3aff6afda7c084a2d06f260 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 15:08:48 +0200 Subject: [PATCH 15/47] light linting --- tools/scrape-juser.py | 95 ++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index e4da082..e75e9cc 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -19,11 +19,16 @@ Usage: > curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%222026%22&sf=author&so=d&rg=&of=xm' > .cache/juser-pubs.xml 2) Cache Person and Publication records from your pool -> dtc get-records $DUMPTHINGS_APIURL public -C XYZPublication > .cache/Publications.jsonl -> dtc get-records $DUMPTHINGS_APIURL public -C XZYPerson > .cache/Person.jsonl +> dtc get-records $DUMPTHINGS_APIURL \ + public -C XYZPublication > .cache/Publications.jsonl +> dtc get-records $DUMPTHINGS_APIURL \ + public -C XZYPerson > .cache/Person.jsonl 3) Invoke the script -> uv run tools/scrape-juser.py --file .cache/juser-pubs.xml --persons /tmp/.cache/Person.jsonl --publications /tmp/.cache/Publications.jsonl +> uv run tools/scrape-juser.py \ + --file .cache/juser-pubs.xml \ + --persons /tmp/.cache/Person.jsonl \ + --publications /tmp/.cache/Publications.jsonl """ import click @@ -59,7 +64,7 @@ def juser_session( def _check_for_immutable_infos( rec: dict, -) -> list: +) -> (dict, list): # get a list of all slots in the record. Treat those without an # attribute about machine-generation as immutable if rec.get('attributes', None) is None: @@ -67,7 +72,7 @@ def _check_for_immutable_infos( do_not_edit = [key for key in rec.keys()] else: generated_infos = \ - [dict['value'] for dict in rec['attributes'][0]['attributes'] \ + [dict['value'] for dict in rec['attributes'][0]['attributes'] if 'importedFrom' in rec['attributes'][0]['predicate']] # don't touch keys if they don't have a machine-generated annotation do_not_edit = [key for key in rec.keys() if key not in generated_infos] @@ -101,7 +106,7 @@ def add_machine_prov( {'predicate': 'http://purl.org/pav/importedFrom', 'value': url, 'attributes': [] - }]} + }]} for value in generated_infos: new = {'predicate': 'prov:generated', 'value': value, @@ -118,7 +123,7 @@ def _lookup( record, field1: str, field2: str -) -> str: +) -> str | None: """Helper function to look up metadata without running into KeyErrors""" parentfield = record.get(field1, None) if parentfield is not None: @@ -175,21 +180,22 @@ class JuserScraper(object): def retrieve_metadata( self, r, - ) -> None: + ): + """For a given MARCXML publication record from Juser, check if a + publication with this DOI is already in the pool. If a record already + exists, check if it needs amendments. If a record does not exist, + assemble a publication metadata record from scratch""" + # TODO: This is spagetti code and should be made prettier # first, get the DOI and check if the publication already exists doi = _lookup(r, '024', 'a') if doi is None: - # When a record has no DOI, we abort. - # TODO: this could check based on other properties, e.g. title + # Abort without DOI. TODO: check based on other properties? return - # Check if a publication with this DOI is already in the pool - # If a record already exists, check if it needs amendments. If a record - # does not exist, assemble a publication metadata record from scratch if doi in self.pubs.keys(): - mrecord, do_not_edit = \ + record, do_not_edit = \ _check_for_immutable_infos(self.pubs[doi]) else: - mrecord = {'schema_type': 'xyzri:XYZPublication', + record = {'schema_type': 'xyzri:XYZPublication', 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), 'identifiers': [ {'schema_type': 'dlthings:DOI', @@ -202,28 +208,28 @@ class JuserScraper(object): juser_id = r.fields[0].data if 'title' not in do_not_edit: title = _lookup(r, '245', 'a') - mrecord['title'] = title + record['title'] = title generated_infos.append('title') if 'description' not in do_not_edit: abstract = _lookup(r, '520', 'a') - mrecord['description'] = abstract + record['description'] = abstract generated_infos.append('description') if 'attributed_to' not in do_not_edit: # add authors. The function returns a list of dicts authors = self._get_authors(r) if len(authors) > 0: - if "attributed_to" not in mrecord.keys(): - mrecord["attributed_to"] = authors + if "attributed_to" not in record.keys(): + record["attributed_to"] = authors else: for author_dict in authors: # check if the author is already part of the record if any( [author_dict['object'] in attr['object'] - for attr in mrecord['attributed_to']] + for attr in record['attributed_to']] ): continue else: - mrecord["attributed_to"].append(author_dict) + record["attributed_to"].append(author_dict) generated_infos.append('attributed_to') if 'generated_by' not in do_not_edit: # generated_by is a list @@ -245,33 +251,33 @@ class JuserScraper(object): "schema_type": "dlthings:Generation", "object": "obo:IAO_0000444"} gen.append(publication_process) - if 'generated_by' not in mrecord.keys(): - mrecord['generated_by'] = gen + if 'generated_by' not in record.keys(): + record['generated_by'] = gen generated_infos.append('generated_by') else: # TODO: figure out how to increment generated_by if some info is # already present pass - if 'rules' not in do_not_edit: + #if 'rules' not in do_not_edit: # rules is a list # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? # TODO: see control vocab at the ZB: https://juser.fz-juelich.de/collection/HGFVOC?ln=en - license = _lookup(r, '915', 'a') + #license = _lookup(r, '915', 'a') # generated_infos.append('rules') # add machine-prov - mrecord = add_machine_prov( + record = add_machine_prov( generated_infos, juser_id, - mrecord + record ) # publication records can be public - self.to_submit_public.append(mrecord) - + self.to_submit_public.append(record) + return def _get_periodical( self, r - ) -> str: + ) -> str | None: """Obtain the ISSN of a publication venue""" # The ZB identifies periodicals via an internal ID, e.g., # 'PERI:(DE-600)2808093-2'We can look up the ISSN @@ -332,7 +338,8 @@ class JuserScraper(object): 'creator': 'ror:04fa4r544', 'notation': orcid}, {'schema_type': 'dlthings:Identifier', - 'creator': 'https://w3id.org/isil/DE-Juel1', # ZB FZJ + 'creator': 'https://w3id.org/isil/DE-Juel1', + # ZB FZJ 'notation': JulID} ], 'attributes': [ @@ -355,13 +362,13 @@ class JuserScraper(object): def _look_up_orcid( self, - JulID: str, + julid: str, ) -> str: - if JulID in self.JulIDs.keys(): + if julid in self.JulIDs.keys(): # this ID is already known, don't query again - return self.JulIDs[JulID] + return self.JulIDs[julid] # URL encode the ID - ID = parse.quote_plus(JulID) + ID = parse.quote_plus(julid) url = \ f'https://juser.fz-juelich.de/search?ln=en&p={ID}&f=&action_search=Search&c=People&sf=author&so=d&rm=&rg=10&sc=0&of=xm' res = self.session.get(url) @@ -369,10 +376,11 @@ class JuserScraper(object): # look up ORCID in the XML above. XML is a mess. try: ORCID = \ - dict((e[0].text.casefold(), e[1].text) for e in x.findall(".//*[@tag='024']"))[ + dict((e[0].text.casefold(), e[1].text) for e in + x.findall(".//*[@tag='024']"))[ 'orcid'] # add to cache - self.JulIDs[JulID] = ORCID + self.JulIDs[julid] = ORCID except KeyError as e: ORCID = None return ORCID @@ -397,8 +405,8 @@ def process_doi(paper: dict) -> str | None: for identifier in paper.get("identifiers", []): if ( - pid_of(identifier.get("creator")) == "ror:01fyxcz70" - or identifier.get("schema_type") == "dlthings:DOI" + pid_of(identifier.get("creator")) == "ror:01fyxcz70" + or identifier.get("schema_type") == "dlthings:DOI" ): return identifier.get("notation") @@ -428,17 +436,20 @@ def process_orcid(person: dict) -> str | None: map_pof_to_pid = { 'G:(DE-HGF)POF4-5251': 'xyzrins:projects/45e542dd-2d2a-41d0-8399-90f2fde72a21', 'G:(DE-HGF)POF4-5252': 'xyzrins:projects/858d0ddb-3c51-4cea-a0e5-a73224b59ae5', - #'G:(DE-HGF)POF4-5253': 'Neuroimaging' # Not relevant for us, not in pool + # 'G:(DE-HGF)POF4-5253': 'Neuroimaging' # Not relevant for us, not in pool 'G:(DE-HGF)POF4-5254': 'xyzrins:projects/5340dd43-6ef0-4cf8-a936-67c2a9f3a725', 'G:(DE-HGF)POF4-5255': 'xyzrins:projects/a3f0a9e0-c945-4e04-a698-be426a9ac075' } + @click.command() -@click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') +@click.option('--dtc-api-url', '-a', + default='https://pool.psychoinformatics.de/api') @click.option('--dtc-collection', '-c', default='public') @click.option('--file', '-f') @click.option('--persons', type=click.File("rb"), default='.cache/Person.jsonl') -@click.option('--publications', type=click.File("rb"), default='.cache/Publications.jsonl') +@click.option('--publications', type=click.File("rb"), + default='.cache/Publications.jsonl') def main( file: str, dtc_api_url: str, -- 2.52.0 From b0160d934ec4a174d6ced5a379f87297ab41a01f Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 2 Jul 2026 15:09:02 +0200 Subject: [PATCH 16/47] fix: actually look up date --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index e75e9cc..d7e6fb4 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -242,7 +242,7 @@ class JuserScraper(object): "object": pof_pid} gen.append(pof_gen) # next, record the publication process - date = _lookup(r, '', '') + date = _lookup(r, '773', 'y') at_location = self._get_periodical(r) if date is not None and at_location is not None: publication_process = \ -- 2.52.0 From ca3c7fbe56bbdb5c6a12d6d9304ea049c355aa58 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 3 Jul 2026 15:45:16 +0200 Subject: [PATCH 17/47] Fix: generated by is a list, can hold several pof topics --- tools/scrape-juser.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index d7e6fb4..be856b9 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -232,15 +232,9 @@ class JuserScraper(object): record["attributed_to"].append(author_dict) generated_infos.append('attributed_to') if 'generated_by' not in do_not_edit: - # generated_by is a list - gen = [] - # first, extract which pof project generated a publication - pof = _lookup(r, '536', 'a') - pof_pid = map_pof_to_pid.get(pof, None) - if pof_pid is not None: - pof_gen = {"schema_type": "dlthings:Generation", - "object": pof_pid} - gen.append(pof_gen) + # first, extract which pof project generated a publication. This may + # be several! If no PoF is found, gen will be an empty list + gen = self._get_pof(r) # next, record the publication process date = _lookup(r, '773', 'y') at_location = self._get_periodical(r) @@ -251,13 +245,14 @@ class JuserScraper(object): "schema_type": "dlthings:Generation", "object": "obo:IAO_0000444"} gen.append(publication_process) - if 'generated_by' not in record.keys(): + if 'generated_by' not in record.keys() and gen is not None: record['generated_by'] = gen generated_infos.append('generated_by') else: # TODO: figure out how to increment generated_by if some info is # already present - pass + record['generated_by'] = gen + generated_infos.append('generated_by') #if 'rules' not in do_not_edit: # rules is a list # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? @@ -274,6 +269,22 @@ class JuserScraper(object): self.to_submit_public.append(record) return + def _get_pof( + self, + r + ) -> list | None: + all_funding = r.get_fields('536') + # for XYZPublication in Research Information, generation is multivalued + pof_gen = [] + for funding in all_funding: + fund = funding['0'] + # this fund may be a PoF association or a grant + pof_pid = map_pof_to_pid.get(fund, None) + if pof_pid is not None: + pof_gen.append({"schema_type": "dlthings:Generation", + "object": pof_pid}) + return pof_gen + def _get_periodical( self, r -- 2.52.0 From f745e930cd2c5660ad8cac0e75ae035b5d848035 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 3 Jul 2026 15:45:56 +0200 Subject: [PATCH 18/47] add content type to record --- tools/scrape-juser.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index be856b9..bf20092 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -253,6 +253,12 @@ class JuserScraper(object): # already present record['generated_by'] = gen generated_infos.append('generated_by') + if 'kind' not in do_not_edit: + content_type = _lookup(r, '336', 'a') + kind = map_content_to_bibitem.get(content_type, None) \ + if content_type is not None else None + if kind is not None: + record['kind'] = kind #if 'rules' not in do_not_edit: # rules is a list # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? @@ -453,6 +459,16 @@ map_pof_to_pid = { } +map_content_to_bibitem = { + 'article': 'bibo:AcademicArticle', + 'lecture': 'fabio:Presentation', + 'Conference Paper': 'bibo:Proceedings', + 'PATENT': 'bibo:Patent', + 'BOOK_CHAPTER': 'bibo:Chapter', + 'Preprint': 'bibo:Manuscript', + 'Output Types/Dissertation': 'bibo:Thesis', +} + @click.command() @click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') -- 2.52.0 From 112340dd7a14fe9382f855d57134eb91d7e6a0a2 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 3 Jul 2026 15:46:19 +0200 Subject: [PATCH 19/47] WIP: Start mapping grant identifiers --- tools/scrape-juser.py | 64 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index bf20092..02698b0 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -455,9 +455,71 @@ map_pof_to_pid = { 'G:(DE-HGF)POF4-5252': 'xyzrins:projects/858d0ddb-3c51-4cea-a0e5-a73224b59ae5', # 'G:(DE-HGF)POF4-5253': 'Neuroimaging' # Not relevant for us, not in pool 'G:(DE-HGF)POF4-5254': 'xyzrins:projects/5340dd43-6ef0-4cf8-a936-67c2a9f3a725', - 'G:(DE-HGF)POF4-5255': 'xyzrins:projects/a3f0a9e0-c945-4e04-a698-be426a9ac075' + 'G:(DE-HGF)POF4-5255': 'xyzrins:projects/a3f0a9e0-c945-4e04-a698-be426a9ac075', } +# TODO: joint work in the pool to get and create necessary infos +map_grant_to_pid = { + 'G:(EU-Grant)101147319': { + 'pid': 'xyzrins:grants/ebrains-2.0', + 'start': '2024-01-01', + 'end': '2026-12-01'}, + 'G:(GEPRIS)552122525': { #SFB B06 + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(GEPRIS)458705014': { # SFB Z03 + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(GEPRIS)458684554': { # SFB C05 + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(GEPRIS)458640473': { # SFB B05 + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(DE-HGF)InterLabs-0015': { # HIBALL + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(EU-Grant)945539': { # HBP SGA3 + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(EU-Grant)604102': { # HBP + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(GEPRIS)431549029': { # SFB 1451 insgesamt + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(DE-Juel1)JL SMHB-2021-2027': { # JL SMHB + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(GEPRIS)524408221': { # Mikrostrukturelle Entwicklung des Gehirns + 'pid': '', + 'start': '', + 'end': '' + }, + 'G:(EU-Grant)826421': { # VirtualBrainCloud + 'pid': '', + 'start': '', + 'end': '' + }, +} map_content_to_bibitem = { 'article': 'bibo:AcademicArticle', -- 2.52.0 From 13069ac76d3bc238557c026df3e6241157bc0225 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 13 Jul 2026 10:22:33 +0200 Subject: [PATCH 20/47] fill in grant details --- tools/scrape-juser.py | 66 +++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 02698b0..3c60ec3 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -465,59 +465,59 @@ map_grant_to_pid = { 'start': '2024-01-01', 'end': '2026-12-01'}, 'G:(GEPRIS)552122525': { #SFB B06 - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/143b2417-20a6-45d1-8140-dfddceb61749', + 'start': '2025-01-01', + 'end': '2028-12-31' }, 'G:(GEPRIS)458705014': { # SFB Z03 - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/1325c9f4-279a-42d1-ba37-ed5577965620', + 'start': '2021-01-01', + 'end': '2028-12-31' }, 'G:(GEPRIS)458684554': { # SFB C05 - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/3977dc9e-59e9-4ff2-bcbf-5bcb8f1c5c2d', + 'start': '2021-01-01', + 'end': '2028-12-31' }, 'G:(GEPRIS)458640473': { # SFB B05 - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/66cbbef2-5fd4-42c6-8348-cb74af435b75', + 'start': '2021-01-01', + 'end': '2028-12-31' }, 'G:(DE-HGF)InterLabs-0015': { # HIBALL - 'pid': '', - 'start': '', - 'end': '' + 'pid': ' https://concepts.datalad.org/s/demo-research-information/ns/grants/391de57c-8040-4cf7-a515-a6791b9a0c89', + 'start': '2020-04-01', + 'end': '2025-03-31' }, 'G:(EU-Grant)945539': { # HBP SGA3 - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/hbp-sga3', + 'start': '2020-04-01', + 'end': '2023-09-30' }, 'G:(EU-Grant)604102': { # HBP - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/ff4d1947-f1ad-4e9f-88fc-57af7b7cfc85', + 'start': '2013-10-01', + 'end': '2017-02-28' }, 'G:(GEPRIS)431549029': { # SFB 1451 insgesamt - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/sfb1451', + 'start': '2021-01-01', + 'end': '2028-12-31' }, 'G:(DE-Juel1)JL SMHB-2021-2027': { # JL SMHB - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/2b40e9ef-d9d2-42f8-a502-2e6efa86a1f7', + 'start': '2021-01-01', + 'end': '2027-12-31' }, 'G:(GEPRIS)524408221': { # Mikrostrukturelle Entwicklung des Gehirns - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/003e63be-a64b-4d72-a912-392b40710a48', + 'start': '2024-10-01', + 'end': '2027-09-30' }, 'G:(EU-Grant)826421': { # VirtualBrainCloud - 'pid': '', - 'start': '', - 'end': '' + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/4919df79-bb48-443a-b74d-3eb83e466187', + 'start': '2018-12-01', + 'end': '2023-05-31' }, } -- 2.52.0 From e8dffac1837094be586d2046f77959ca48487fc7 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 13 Jul 2026 10:59:27 +0200 Subject: [PATCH 21/47] add ABCD-J --- tools/scrape-juser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 3c60ec3..2ab61c9 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -519,6 +519,11 @@ map_grant_to_pid = { 'start': '2018-12-01', 'end': '2023-05-31' }, + 'G:(MKW NRW)KP22-106A': { # ABCD-J + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/mkw-abcd-j', + 'start': '2023-05-01', + 'end': '2027-04-30', + } } map_content_to_bibitem = { -- 2.52.0 From e5bd0eaaadae22dc2f5291d67f8fa4a561e277cb Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 13 Jul 2026 11:05:44 +0200 Subject: [PATCH 22/47] WIP: adding funding acks incl. dates --- tools/scrape-juser.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 2ab61c9..67c540e 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -259,6 +259,22 @@ class JuserScraper(object): if content_type is not None else None if kind is not None: record['kind'] = kind + if 'influenced_by' not in do_not_edit: + funding = self._get_funding(r) + if 'influenced_by' not in record.keys(): + record['influenced_by'] = funding + else: + # check if the funding is already part of the record + # TODO: factor out this pattern + for fund_dict in funding: + if any( + [fund_dict['object'] in attr['object'] + for attr in record['influenced_by']] + ): + continue + else: + record["influenced_by"].append(fund_dict) + #if 'rules' not in do_not_edit: # rules is a list # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? @@ -291,6 +307,35 @@ class JuserScraper(object): "object": pof_pid}) return pof_gen + def _get_funding( + self, + r, + ) -> list | None: + all_funding = r.get_fields('536') + funding_ack = [] + for funding in all_funding: + fund = funding['0'] + # this fund may be a PoF association or a grant + fund_rec = map_grant_to_pid.get(fund, None) + if fund_rec is not None: + funding_ack.append( + { "schema_type": "xyzri:XYZInfluence", + "object": fund_rec['pid'], + "roles": [ + "FRAPO:Funding" + ], + "ended": { + "at_time": fund_rec['ended'], + "schema_type": "dlthings:End" + }, + "started": { + "at_time": fund_rec['started'], + "schema_type": "dlthings:Start" + }, + } + ) + return funding_ack + def _get_periodical( self, r -- 2.52.0 From 20eb78bfe779dcac6fccb44421d726e9f1560731 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 17 Jul 2026 06:15:42 +0200 Subject: [PATCH 23/47] fix example --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 67c540e..2ed861d 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -22,7 +22,7 @@ Usage: > dtc get-records $DUMPTHINGS_APIURL \ public -C XYZPublication > .cache/Publications.jsonl > dtc get-records $DUMPTHINGS_APIURL \ - public -C XZYPerson > .cache/Person.jsonl + public -C XYZPerson > .cache/Person.jsonl 3) Invoke the script > uv run tools/scrape-juser.py \ -- 2.52.0 From 10414ac754b23bd810e933eba725b06ce50b7cba Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 17 Jul 2026 06:16:39 +0200 Subject: [PATCH 24/47] fix: only add funding if info is available --- tools/scrape-juser.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 2ed861d..2af6c2e 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -261,19 +261,20 @@ class JuserScraper(object): record['kind'] = kind if 'influenced_by' not in do_not_edit: funding = self._get_funding(r) - if 'influenced_by' not in record.keys(): - record['influenced_by'] = funding - else: - # check if the funding is already part of the record - # TODO: factor out this pattern - for fund_dict in funding: - if any( - [fund_dict['object'] in attr['object'] - for attr in record['influenced_by']] - ): - continue - else: - record["influenced_by"].append(fund_dict) + if funding: + if 'influenced_by' not in record.keys(): + record['influenced_by'] = funding + else: + # check if the funding is already part of the record + # TODO: factor out this pattern + for fund_dict in funding: + if any( + [fund_dict['object'] in attr['object'] + for attr in record['influenced_by']] + ): + continue + else: + record["influenced_by"].append(fund_dict) #if 'rules' not in do_not_edit: # rules is a list -- 2.52.0 From 46f6216e247676fea4f1e501b15a8426c2518a8b Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 17 Jul 2026 06:17:08 +0200 Subject: [PATCH 25/47] fix: keynames --- tools/scrape-juser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 2af6c2e..ab520a6 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -326,11 +326,11 @@ class JuserScraper(object): "FRAPO:Funding" ], "ended": { - "at_time": fund_rec['ended'], + "at_time": fund_rec['end'], "schema_type": "dlthings:End" }, "started": { - "at_time": fund_rec['started'], + "at_time": fund_rec['start'], "schema_type": "dlthings:Start" }, } -- 2.52.0 From ee48f9bdf020c54793467be34179935c320d87e5 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 17 Jul 2026 06:18:18 +0200 Subject: [PATCH 26/47] add pof4 neuroimaging its not relevant for us strictly, but publications often use it nevertheless --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index ab520a6..14182bc 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -499,7 +499,7 @@ def process_orcid(person: dict) -> str | None: map_pof_to_pid = { 'G:(DE-HGF)POF4-5251': 'xyzrins:projects/45e542dd-2d2a-41d0-8399-90f2fde72a21', 'G:(DE-HGF)POF4-5252': 'xyzrins:projects/858d0ddb-3c51-4cea-a0e5-a73224b59ae5', - # 'G:(DE-HGF)POF4-5253': 'Neuroimaging' # Not relevant for us, not in pool + 'G:(DE-HGF)POF4-5253': 'xyzrins:projects/64777d99-26f6-441c-9849-ba8018571de8', 'G:(DE-HGF)POF4-5254': 'xyzrins:projects/5340dd43-6ef0-4cf8-a936-67c2a9f3a725', 'G:(DE-HGF)POF4-5255': 'xyzrins:projects/a3f0a9e0-c945-4e04-a698-be426a9ac075', } -- 2.52.0 From c4f1e690e649239a6c1c38c0a006e412a8d3e8ad Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 17 Jul 2026 15:27:33 +0200 Subject: [PATCH 27/47] safeguard doi lookup --- tools/scrape-juser.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 14182bc..3ad4196 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -187,7 +187,7 @@ class JuserScraper(object): assemble a publication metadata record from scratch""" # TODO: This is spagetti code and should be made prettier # first, get the DOI and check if the publication already exists - doi = _lookup(r, '024', 'a') + doi = self._get_doi(r) if doi is None: # Abort without DOI. TODO: check based on other properties? return @@ -292,6 +292,19 @@ class JuserScraper(object): self.to_submit_public.append(record) return + def _get_doi( + self, + r + ) -> str | None: + doi = None + # check if field 024 exists + test = _lookup(r, '024', 'a') + if test: + if r['024'].subfields[1].value == 'doi': + doi = r['024']['a'] + return doi + + def _get_pof( self, r -- 2.52.0 From 27d5152395e1b412d70e8173e315f4bece62a2be Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 17 Jul 2026 15:27:53 +0200 Subject: [PATCH 28/47] no license lookup --- tools/scrape-juser.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 3ad4196..54ce329 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -275,13 +275,6 @@ class JuserScraper(object): continue else: record["influenced_by"].append(fund_dict) - - #if 'rules' not in do_not_edit: - # rules is a list - # TODO: This needs a license lookup/mapping. Maybe from enrich-via-doi.py? - # TODO: see control vocab at the ZB: https://juser.fz-juelich.de/collection/HGFVOC?ln=en - #license = _lookup(r, '915', 'a') - # generated_infos.append('rules') # add machine-prov record = add_machine_prov( generated_infos, -- 2.52.0 From 68f07b1249a353426215b7e3d1795bd2687299fa Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Sat, 18 Jul 2026 17:14:21 +0200 Subject: [PATCH 29/47] Add submission --- tools/scrape-juser.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 54ce329..bd0c463 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -43,6 +43,8 @@ from urllib import parse from requests import Session from xml.etree import ElementTree as ET +from dump_things_pyclient.communicate import collection_write_record + def juser_session( url: str = 'https://juser.fz-juelich.de/youraccount/login', @@ -171,10 +173,29 @@ class JuserScraper(object): self, ) -> None: # finally, submit: - # TODO IMPLEMENT ME! + print('DONE!') print([json.dumps(r) for r in self.to_submit_protected]) print([json.dumps(r) for r in self.to_submit_public]) - + for record in self.to_submit_public: + print(f"submitting public record with pid {record['pid']}...") + collection_write_record( + service_url=self.pool, + collection='public', + class_name='XYZPublication', + record=record, + format='json', + token=environ['DTC_TOKEN'] + ) + for record in self.to_submit_protected: + print(f"submitting protected record with pid {record['pid']}...") + collection_write_record( + service_url=self.pool, + collection='protected', + class_name='XYZPerson', + record=record, + format='json', + token=environ['DTC_TOKEN'] + ) return def retrieve_metadata( -- 2.52.0 From 2ce5a69c9e9ff551411a010c8850a9b0dd6ea1f5 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Sat, 18 Jul 2026 17:14:45 +0200 Subject: [PATCH 30/47] fix HIBALL pid --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index bd0c463..3739065 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -558,7 +558,7 @@ map_grant_to_pid = { 'end': '2028-12-31' }, 'G:(DE-HGF)InterLabs-0015': { # HIBALL - 'pid': ' https://concepts.datalad.org/s/demo-research-information/ns/grants/391de57c-8040-4cf7-a515-a6791b9a0c89', + 'pid': 'https://concepts.datalad.org/s/demo-research-information/ns/grants/391de57c-8040-4cf7-a515-a6791b9a0c89', 'start': '2020-04-01', 'end': '2025-03-31' }, -- 2.52.0 From 4c567c11384a5d3b4ebe9a7910a8fbf3cc01744d Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 20 Jul 2026 10:53:06 +0200 Subject: [PATCH 31/47] Cache protected person records in addition --- .forgejo/workflows/scrape-juser.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/scrape-juser.yml b/.forgejo/workflows/scrape-juser.yml index d9de6bf..b2f24fe 100644 --- a/.forgejo/workflows/scrape-juser.yml +++ b/.forgejo/workflows/scrape-juser.yml @@ -38,6 +38,7 @@ jobs: mkdir .cache dtc get-records $DUMPTHINGS_APIURL public -C XYZPublication > .cache/Publications.jsonl dtc get-records $DUMPTHINGS_APIURL public -C XZYPerson > .cache/Person.jsonl + dtc get-records $DUMPTHINGS_APIURL protected -C XZYPerson >> .cache/Person.jsonl - name: Pre-fetch Juser records curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%22${{ inputs.year }}%22&sf=author&so=d&rg=&of=xm' > .cache/juser-pubs.xml -- 2.52.0 From 226d6c624ea63465c5325b65b1812b874abc1bc7 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 20 Jul 2026 10:53:56 +0200 Subject: [PATCH 32/47] stop dumping records, list new publications --- tools/scrape-juser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 3739065..8c1d74d 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -173,9 +173,8 @@ class JuserScraper(object): self, ) -> None: # finally, submit: - print('DONE!') - print([json.dumps(r) for r in self.to_submit_protected]) - print([json.dumps(r) for r in self.to_submit_public]) + #print([json.dumps(r) for r in self.to_submit_protected]) + #print([json.dumps(r) for r in self.to_submit_public]) for record in self.to_submit_public: print(f"submitting public record with pid {record['pid']}...") collection_write_record( @@ -216,12 +215,13 @@ class JuserScraper(object): record, do_not_edit = \ _check_for_immutable_infos(self.pubs[doi]) else: + print('CREATING NEW PUBLICATION FOR DOI ', doi) record = {'schema_type': 'xyzri:XYZPublication', 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), 'identifiers': [ {'schema_type': 'dlthings:DOI', 'notation': doi}] - } + } do_not_edit = [] generated_infos = [] # record of the juser ID, for machine provenance and linking to the -- 2.52.0 From 80b7405b165c86162a107c55d3c64d1f1b7c357b Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 20 Jul 2026 15:06:26 +0200 Subject: [PATCH 33/47] split public and protected submissions --- tools/scrape-juser.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 8c1d74d..45c506b 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -171,30 +171,22 @@ class JuserScraper(object): def submit_records( self, + records: list, + collection: str, + class_name: str, ) -> None: # finally, submit: - #print([json.dumps(r) for r in self.to_submit_protected]) - #print([json.dumps(r) for r in self.to_submit_public]) - for record in self.to_submit_public: - print(f"submitting public record with pid {record['pid']}...") + for record in records: + print(f"submitting record with pid {record['pid']}" + f" to collection {collection}") collection_write_record( service_url=self.pool, - collection='public', - class_name='XYZPublication', + collection=collection, + class_name=class_name, record=record, format='json', token=environ['DTC_TOKEN'] ) - for record in self.to_submit_protected: - print(f"submitting protected record with pid {record['pid']}...") - collection_write_record( - service_url=self.pool, - collection='protected', - class_name='XYZPerson', - record=record, - format='json', - token=environ['DTC_TOKEN'] - ) return def retrieve_metadata( @@ -617,10 +609,14 @@ map_content_to_bibitem = { @click.option('--persons', type=click.File("rb"), default='.cache/Person.jsonl') @click.option('--publications', type=click.File("rb"), default='.cache/Publications.jsonl') +@click.option('--submit', + type=click.Choice(['XYZPerson', 'XYZPublication', 'both']), + default='both') def main( file: str, dtc_api_url: str, dtc_collection: str, + submit: str, persons, publications, ) -> None: @@ -650,9 +646,18 @@ def main( pubs=all_pubs, ) JS.create_records() - # TODO: potentially do reporting about the records now - JS.submit_records() - + if submit in ['both', 'XYZPublication']: + JS.submit_records( + JS.to_submit_public, + collection='public', + class_name='XYZPublication' + ) + if submit in ['both', 'XYZPerson']: + JS.submit_records( + JS.to_submit_protected, + collection='protected', + class_name='XYZPerson' + ) #if __name__ == '__main__': # main() -- 2.52.0 From 4b1dc0d45f6074cbae655e7a0457304d013a51c1 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Mon, 20 Jul 2026 15:06:46 +0200 Subject: [PATCH 34/47] factor out common variable --- tools/scrape-juser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 45c506b..21c916d 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -393,8 +393,9 @@ class JuserScraper(object): # if the author has an FZJ affiliation, get their info if '(DE-Juel1)' in author.subfields[1].value: JulID = author.subfields[1].value + display_label = author.subfields[0].value logging.info('Found author {} under JulID {}'.format( - author.subfields[0].value, + display_label, JulID )) orcid = self._look_up_orcid(JulID) @@ -407,6 +408,7 @@ class JuserScraper(object): # we don't pursue this further continue if pid is None: + print('CREATING NEW PERSON RECORD FOR ', display_label) p_pid = 'xyzrins:persons/' + str(uuid.uuid4()) p_rec = {'pid': p_pid, # MARCXML does only report author names as one string @@ -414,7 +416,7 @@ class JuserScraper(object): # etc). Parsing this into first name and family name # is error-prone. Instead, we use their string as a # display label - 'display_label': author.subfields[0].value, + 'display_label': display_label, 'identifiers': [ {'schema_type': 'xyzri:ORCID', 'creator': 'ror:04fa4r544', -- 2.52.0 From d17c03e83d830bd6fed10535bf05fe465118e7c1 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 23 Jul 2026 14:01:50 +0200 Subject: [PATCH 35/47] refactor and restructure records source document: https://hedgedoc.psychoinformatics.de/3cSouq0YSJ6m64_ArWpJEg?view --- tools/scrape-juser.py | 635 ++++++++++++++++++++++++------------------ 1 file changed, 367 insertions(+), 268 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 21c916d..d4b8681 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -34,6 +34,7 @@ Usage: import click import json import logging +import requests import uuid from collections.abc import Callable from pymarc import parse_xml_to_array @@ -64,63 +65,6 @@ def juser_session( return s -def _check_for_immutable_infos( - rec: dict, -) -> (dict, list): - # get a list of all slots in the record. Treat those without an - # attribute about machine-generation as immutable - if rec.get('attributes', None) is None: - # no attributes present -- all existing keys are human annotated! - do_not_edit = [key for key in rec.keys()] - else: - generated_infos = \ - [dict['value'] for dict in rec['attributes'][0]['attributes'] - if 'importedFrom' in rec['attributes'][0]['predicate']] - # don't touch keys if they don't have a machine-generated annotation - do_not_edit = [key for key in rec.keys() if key not in generated_infos] - return rec, do_not_edit - - -def add_or_edit_if_mutable( - mrecord, - recordkey: str, - recordvalue: str, - do_not_edit: list -) -> dict: - if recordkey in do_not_edit: - return mrecord - if recordvalue is not None: - mrecord[recordkey] = recordvalue - return mrecord - - -# TODO: from scrape-calendar.py -- factor out and import! -def add_machine_prov( - generated_infos, - juser_id, - rec, - scriptpid='xyzrins:instruments/d3126ff5-623c-48af-ac48-4d921ef9b80d' -): - if generated_infos is None: - return rec - url = 'https://juser.fz-juelich.de/record/' + juser_id - prov = {'attributes': [ - {'predicate': 'http://purl.org/pav/importedFrom', - 'value': url, - 'attributes': [] - }]} - for value in generated_infos: - new = {'predicate': 'prov:generated', - 'value': value, - 'characterized_by': [{ - 'predicate': 'prov:generated_by', - 'object': scriptpid - }]} - prov['attributes'][0]['attributes'].append(new) - rec.update(prov) - return rec - - def _lookup( record, field1: str, @@ -157,6 +101,8 @@ class JuserScraper(object): self.to_submit_protected = [] # establish a session to Juser self.session = juser_session() + # potentially make this a parameter + self.scriptpid = 'xyzrins:instruments/d3126ff5-623c-48af-ac48-4d921ef9b80d' def create_records( self @@ -179,14 +125,300 @@ class JuserScraper(object): for record in records: print(f"submitting record with pid {record['pid']}" f" to collection {collection}") - collection_write_record( - service_url=self.pool, - collection=collection, - class_name=class_name, - record=record, - format='json', - token=environ['DTC_TOKEN'] - ) + try: + collection_write_record( + service_url=self.pool, + collection=collection, + class_name=class_name, + record=record, + format='json', + token=environ['DTC_TOKEN'] + ) + except requests.exceptions.HTTPError: + print("SUBMISSION ERROR FOR RECORD: ") + print(json.dumps(record)) + return + + def _check_if_mutable( + self, + record: dict, + k: str, + only_self_edits: bool = False, + predicate: str | None = None, + ) -> bool: + """If a record either does not already have the info, or the info is + annotated to be machine-generated, allow overwriting it. + params: + record: dict -> the metadata record + k: str -> record key to check + only_self_edits: True|False -> only allows overwriting existing infos if + they stem from the same script + """ + + if k not in record.keys(): + return True + infos = record[k] + if type(infos) == dict and 'annotations' in infos.keys(): + importedBy = \ + infos.get('annotations', {}).get('http://purl.org/pav/importedBy', 'nothing') + return self._is_machine_generated(importedBy, only_self_edits) + # if the key is a data property, infos is just a string. + attributes = record.get('attributes', [{}]) + for attribute in attributes: + if attribute.get('predicate', None) == predicate: + return \ + self._is_machine_generated(attribute.get('annotations', {}).get('http://purl.org/pav/importedBy', 'nothing')) + + def _is_machine_generated( + self, + importedBy, + only_self_edits: bool = False, + ): + try: + if importedBy.startswith('xyzrins:instruments'): + if not only_self_edits: + return True + else: + return importedBy == self.scriptpid + except AttributeError: + import pdb; pdb.set_trace() + return False + + def _get_data_property( + self, + record: dict, + prop: str, + predicate: str, + lookup: tuple + ): + if self._check_if_mutable(record, prop, predicate=predicate): + return _lookup(lookup[0], lookup[1], lookup[2]) + return None + + def _get_attribution( + self, + record, + r, + annotations + ) -> dict: + authors = r.get_fields('700') + new_attributions = [] + for author in authors: + orcid = None + # if the author haus an FZJ affiliation, get their info. We can most + # reliably look up Orcid based on Julich ID -- external authors + # often are only mentioned by name + if '(DE-Juel1)' in author.subfields[1].value: + JulID = author.subfields[1].value + display_label = author.subfields[0].value + orcid, record_id = self._look_up_orcid(JulID) + logging.info(f'Found author {display_label} under ID {JulID} ' + f'with orcid {orcid}: ' + f'https://juser.fz-juelich.de/record/{record_id}') + if orcid is None: + # stop if there is no ORCID for a Juelich-based author + continue + # either get the pid of the preexisting record or build new pid + pid = self.persons.get(orcid, {}).get('pid', + 'xyzrins:persons/' + str(uuid.uuid4())) + person_annotations = \ + {'http://purl.org/pav/importedBy': self.scriptpid, + 'http://purl.org/pav/importedFrom': + f'https://juser.fz-juelich.de/record/{record_id}'} + # create the author attribution: + attr = {"schema_type": "dlthings:Attribution", + "object": pid, + "annotations": annotations} + # submit new person record, if not yet existing + if orcid not in self.persons.keys(): + self._new_person( + p_pid=pid, + display_label=display_label, + orcid=orcid, + JulID=JulID, + annotations=person_annotations + ) + # check if the author is already attributed in the record + for i, attribution in enumerate(record.get('attributed_to', [{}])): + found = False + if pid == attribution.get('object'): + # the author is known. Check if editable, if so, add author + # and annotation to the record: + if self._is_machine_generated( + attribution.get('annotations', {}).get('http://purl.org/pav/importedBy', 'nothing')): + if attribution != attr: + # only update if the record differs + record['attributed_to'][i] = attr + # search is over once we found the author, continue next + found = True + continue + if not found: + new_attributions.append(attr) + old_attributions = record.get('attributed_to', []) + old_attributions.extend(new_attributions) + return old_attributions + + def _new_person( + self, + p_pid, + display_label, + orcid, + JulID, + annotations + ): + print('CREATING NEW PERSON RECORD FOR ', display_label) + p_rec = {'pid': p_pid, + # MARCXML does only report author names as one string + # with varying formatting (incl. or excl. initials, + # etc). Parsing this into first name and family name + # is error-prone. Instead, we use their string as a + # display label + 'display_label': display_label, + 'identifiers': [ + {'schema_type': 'xyzri:ORCID', + 'creator': 'ror:04fa4r544', + 'notation': orcid, + 'annotations': annotations}, + {'schema_type': 'dlthings:Identifier', + 'creator': 'https://w3id.org/isil/DE-Juel1', + # ZB FZJ + 'notation': JulID, + 'annotations': annotations}], + } + # collect to be created record for submission. Person records + # are protected + self.to_submit_protected.append(p_rec) + # store the author in internal cache to not resubmit + self.persons[orcid] = p_rec + return + + def _get_generation( + self, + record, + r, + annotations + ): + generated_by = record.get('generated_by', []) + new_generations = [] + # extract which pof project generated a publication. This may + # be several! If no PoF is found, gen will be an empty list + pof = self._get_pof(r, annotations) + if pof: + for p in pof: + # is there an existing record, and can we edit it? + pof_pid = p['object'] + found = False + for i, generation in enumerate(generated_by): + found = False + if generation.get('object', None) == pof_pid: + found = True + if self._is_machine_generated( + generation.get('annotations', {}).get('http://purl.org/pav/importedBy', None)): + if p != generation: + # overwrite + generated_by[i] = p + if not found: + new_generations.append(p) + # next, record the publication process + date = _lookup(r, '773', 'y') + at_location = self._get_periodical(r) + if date is not None and at_location is not None: + publication_process = \ + {"at_location": "ISSN:{}".format(at_location), + "at_time": "{}".format(date), + "schema_type": "dlthings:Generation", + "object": "obo:IAO_0000444", + "annotations": annotations} + # is there an existing record, and can we edit it? + periodical_pid = publication_process['object'] + for i, generation in enumerate(generated_by): + found = False + if generation.get('object', 'nothing') == periodical_pid: + if self._is_machine_generated( + generation.get('annotations', {}).get('http://purl.org/pav/importedBy', 'nothing')): + if publication_process != generation: + # overwrite + generated_by[i] = publication_process + found = True + if not found: + new_generations.append(publication_process) + generated_by.extend(new_generations) + return generated_by + + def _get_influence( + self, + record, + r, + annotations + ): + influenced_by = record.get('influenced_by', []) + new_influences = [] + funding = self._get_funding(r, annotations) + if funding: + for fund in funding: + # check if the funding is already part of the record + fund_pid = fund.get('object', None) + for i, influence in enumerate(influenced_by): + found = False + if influence.get('object', 'nothing') == fund_pid: + if self._is_machine_generated( + influence.get('annotations', {}).get('http://purl.org/pav/importedBy', None)): + if fund != influence: + # overwrite + influenced_by[i] = fund + found = True + if not found: + new_influences.append(fund) + influenced_by.extend(new_influences) + return influenced_by + + def _get_pof( + self, + r, + annotations + ) -> list | None: + all_funding = r.get_fields('536') + # for XYZPublication in Research Information, generation is multivalued + pof_gen = [] + for funding in all_funding: + fund = funding['0'] + # this fund may be a PoF association or a grant + pof_pid = map_pof_to_pid.get(fund, None) + if pof_pid is not None: + pof_gen.append({"schema_type": "dlthings:Generation", + "object": pof_pid, + "annotations": annotations}) + return pof_gen + + def _get_funding( + self, + r, + annotations, + ) -> list | None: + all_funding = r.get_fields('536') + funding_ack = [] + for funding in all_funding: + fund = funding['0'] + # this fund may be a PoF association or a grant + fund_rec = map_grant_to_pid.get(fund, None) + if fund_rec is not None: + funding_ack.append( + {"schema_type": "xyzri:XYZInfluence", + "object": fund_rec['pid'], + "roles": ["FRAPO:Funding"], + "ended": {"at_time": fund_rec['end'], + "schema_type": "dlthings:End"}, + "started": {"at_time": fund_rec['start'], + "schema_type": "dlthings:Start"}, + "annotations": annotations}) + return funding_ack + + def _update_person( + self, + annotations + ): + """e.g., add JulID""" + # TODO return def retrieve_metadata( @@ -197,103 +429,72 @@ class JuserScraper(object): publication with this DOI is already in the pool. If a record already exists, check if it needs amendments. If a record does not exist, assemble a publication metadata record from scratch""" - # TODO: This is spagetti code and should be made prettier # first, get the DOI and check if the publication already exists doi = self._get_doi(r) if doi is None: - # Abort without DOI. TODO: check based on other properties? return - if doi in self.pubs.keys(): - record, do_not_edit = \ - _check_for_immutable_infos(self.pubs[doi]) - else: - print('CREATING NEW PUBLICATION FOR DOI ', doi) - record = {'schema_type': 'xyzri:XYZPublication', - 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), - 'identifiers': [ - {'schema_type': 'dlthings:DOI', - 'notation': doi}] - } - do_not_edit = [] - generated_infos = [] # record of the juser ID, for machine provenance and linking to the # original juser_id = r.fields[0].data - if 'title' not in do_not_edit: - title = _lookup(r, '245', 'a') - record['title'] = title - generated_infos.append('title') - if 'description' not in do_not_edit: - abstract = _lookup(r, '520', 'a') - record['description'] = abstract - generated_infos.append('description') - if 'attributed_to' not in do_not_edit: - # add authors. The function returns a list of dicts - authors = self._get_authors(r) - if len(authors) > 0: - if "attributed_to" not in record.keys(): - record["attributed_to"] = authors + annotations = {"obo:NCIT_C42704": self.scriptpid, + "obo:NCIT_P378": f'https://juser.fz-juelich.de/record/{juser_id}'} + if doi in self.pubs.keys(): + record = self.pubs[doi] + else: + print('CREATING NEW PUBLICATION FOR DOI ', doi) + record = {'schema_type': 'xyzri:XYZPublication', + 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), + 'identifiers': [ + {'schema_type': 'dlthings:DOI', + 'notation': doi, + 'annotations': annotations} + ] + } + # the next steps obtain metadata fields IF the record allows overwriting + title = \ + self._get_data_property(record, 'title', 'dcterms:title', + (r, '245', 'a')) + description = \ + self._get_data_property(record, 'description', 'dcterms:abstract', + (r, '520', 'a')) + kind = \ + self._get_data_property(record, 'kind', 'dcterms:type', + (r, '336', 'a')) + content_type = map_content_to_bibitem.get(kind, None) \ + if kind is not None else None + kind = content_type + if kind == 'MISC': + import pdb; pdb.set_trace() + # add empty attributes for machine prov for data properties + # TODO: the slot could also be determined by a different script + attributes = record.get('attributes', []) + for prop, slot, term in [(title, 'title', 'dcterms:title'), + (description, 'description', 'dcterms:abstract'), + (kind, 'kind', 'dcterms:type')]: + if prop is not None: + record[slot] = prop + metadata = {'predicate': term, + 'value': prop, + 'annotations': annotations} + if attributes: + for i, attr in enumerate(attributes): + found = False + if attr.get('predicate', None) == term: + attributes[i] = metadata + if not found: + attributes.append(metadata) else: - for author_dict in authors: - # check if the author is already part of the record - if any( - [author_dict['object'] in attr['object'] - for attr in record['attributed_to']] - ): - continue - else: - record["attributed_to"].append(author_dict) - generated_infos.append('attributed_to') - if 'generated_by' not in do_not_edit: - # first, extract which pof project generated a publication. This may - # be several! If no PoF is found, gen will be an empty list - gen = self._get_pof(r) - # next, record the publication process - date = _lookup(r, '773', 'y') - at_location = self._get_periodical(r) - if date is not None and at_location is not None: - publication_process = \ - {"at_location": "ISSN:{}".format(at_location), - "at_time": "{}".format(date), - "schema_type": "dlthings:Generation", - "object": "obo:IAO_0000444"} - gen.append(publication_process) - if 'generated_by' not in record.keys() and gen is not None: - record['generated_by'] = gen - generated_infos.append('generated_by') - else: - # TODO: figure out how to increment generated_by if some info is - # already present - record['generated_by'] = gen - generated_infos.append('generated_by') - if 'kind' not in do_not_edit: - content_type = _lookup(r, '336', 'a') - kind = map_content_to_bibitem.get(content_type, None) \ - if content_type is not None else None - if kind is not None: - record['kind'] = kind - if 'influenced_by' not in do_not_edit: - funding = self._get_funding(r) - if funding: - if 'influenced_by' not in record.keys(): - record['influenced_by'] = funding - else: - # check if the funding is already part of the record - # TODO: factor out this pattern - for fund_dict in funding: - if any( - [fund_dict['object'] in attr['object'] - for attr in record['influenced_by']] - ): - continue - else: - record["influenced_by"].append(fund_dict) - # add machine-prov - record = add_machine_prov( - generated_infos, - juser_id, - record - ) + attributes.append(metadata) + record['attributes'] = attributes + + attributed_to = self._get_attribution(record, r, annotations) + generated_by = self._get_generation(record, r, annotations) + influenced_by = self._get_influence(record, r, annotations) + for prop, slot in [(attributed_to, 'attributed_to'), + (generated_by, 'generated_by'), + (influenced_by, 'influenced_by')]: + if prop: + record[slot] = prop # publication records can be public self.to_submit_public.append(record) return @@ -310,52 +511,6 @@ class JuserScraper(object): doi = r['024']['a'] return doi - - def _get_pof( - self, - r - ) -> list | None: - all_funding = r.get_fields('536') - # for XYZPublication in Research Information, generation is multivalued - pof_gen = [] - for funding in all_funding: - fund = funding['0'] - # this fund may be a PoF association or a grant - pof_pid = map_pof_to_pid.get(fund, None) - if pof_pid is not None: - pof_gen.append({"schema_type": "dlthings:Generation", - "object": pof_pid}) - return pof_gen - - def _get_funding( - self, - r, - ) -> list | None: - all_funding = r.get_fields('536') - funding_ack = [] - for funding in all_funding: - fund = funding['0'] - # this fund may be a PoF association or a grant - fund_rec = map_grant_to_pid.get(fund, None) - if fund_rec is not None: - funding_ack.append( - { "schema_type": "xyzri:XYZInfluence", - "object": fund_rec['pid'], - "roles": [ - "FRAPO:Funding" - ], - "ended": { - "at_time": fund_rec['end'], - "schema_type": "dlthings:End" - }, - "started": { - "at_time": fund_rec['start'], - "schema_type": "dlthings:Start" - }, - } - ) - return funding_ack - def _get_periodical( self, r @@ -380,77 +535,14 @@ class JuserScraper(object): ISSN = None return ISSN - def _get_authors( - self, - r, - ) -> list[dict]: - """Obtain the ORCIDs of authors from a publication that - come from the FZJ""" - attributed_to = [] - authors = r.get_fields('700') - for author in authors: - orcid = None - # if the author has an FZJ affiliation, get their info - if '(DE-Juel1)' in author.subfields[1].value: - JulID = author.subfields[1].value - display_label = author.subfields[0].value - logging.info('Found author {} under JulID {}'.format( - display_label, - JulID - )) - orcid = self._look_up_orcid(JulID) - logging.info('their orcid is {}'.format(orcid)) - pid = self.persons[orcid]['pid'] if \ - orcid is not None and orcid in self.persons.keys() \ - else None - if orcid is None: - # if the author is not from Jülich or there is no ORCID, - # we don't pursue this further - continue - if pid is None: - print('CREATING NEW PERSON RECORD FOR ', display_label) - p_pid = 'xyzrins:persons/' + str(uuid.uuid4()) - p_rec = {'pid': p_pid, - # MARCXML does only report author names as one string - # with varying formatting (incl. or excl. initials, - # etc). Parsing this into first name and family name - # is error-prone. Instead, we use their string as a - # display label - 'display_label': display_label, - 'identifiers': [ - {'schema_type': 'xyzri:ORCID', - 'creator': 'ror:04fa4r544', - 'notation': orcid}, - {'schema_type': 'dlthings:Identifier', - 'creator': 'https://w3id.org/isil/DE-Juel1', - # ZB FZJ - 'notation': JulID} - ], - 'attributes': [ - {'predicate': 'http://purl.org/pav/importedFrom', - 'value': 'xyzrins:instruments/d3126ff5-623c-48af-ac48-4d921ef9b80d'} - ] - } - # collect to be created record for submission. Person records - # are protected - self.to_submit_protected.append(p_rec) - # store the author in internal cache to not resubmit - self.persons[orcid] = p_rec - attr = {"schema_type": "dlthings:Attribution", - "object": p_pid} - else: - attr = {"schema_type": "dlthings:Attribution", - "object": pid} - attributed_to.append(attr) - return attributed_to def _look_up_orcid( self, julid: str, - ) -> str: + ) -> (str, str): if julid in self.JulIDs.keys(): # this ID is already known, don't query again - return self.JulIDs[julid] + return self.JulIDs[julid], None # URL encode the ID ID = parse.quote_plus(julid) url = \ @@ -467,7 +559,12 @@ class JuserScraper(object): self.JulIDs[julid] = ORCID except KeyError as e: ORCID = None - return ORCID + try: + record_id = [e.text for e in x.findall(".//*[@tag='001']")][0] + record_source = f'https://juser.fz-juelich.de/record/{record_id}' + except Exception as e: + record_source = None + return ORCID, record_source def map_record_to_feature( @@ -597,12 +694,14 @@ map_content_to_bibitem = { 'article': 'bibo:AcademicArticle', 'lecture': 'fabio:Presentation', 'Conference Paper': 'bibo:Proceedings', + 'CONFERENCE_PAPER': 'bibo:Proceedings', 'PATENT': 'bibo:Patent', 'BOOK_CHAPTER': 'bibo:Chapter', 'Preprint': 'bibo:Manuscript', 'Output Types/Dissertation': 'bibo:Thesis', } + @click.command() @click.option('--dtc-api-url', '-a', default='https://pool.psychoinformatics.de/api') @@ -661,5 +760,5 @@ def main( class_name='XYZPerson' ) -#if __name__ == '__main__': -# main() +if __name__ == '__main__': + main() -- 2.52.0 From d209e1ba26cc8c00de5850d51a8851ac8562edbe Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 23 Jul 2026 15:08:04 +0200 Subject: [PATCH 36/47] Fix: don't create duplicate author entries --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index d4b8681..4632b3c 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -204,6 +204,7 @@ class JuserScraper(object): authors = r.get_fields('700') new_attributions = [] for author in authors: + found = False orcid = None # if the author haus an FZJ affiliation, get their info. We can most # reliably look up Orcid based on Julich ID -- external authors @@ -240,7 +241,6 @@ class JuserScraper(object): ) # check if the author is already attributed in the record for i, attribution in enumerate(record.get('attributed_to', [{}])): - found = False if pid == attribution.get('object'): # the author is known. Check if editable, if so, add author # and annotation to the record: -- 2.52.0 From 1ce2ef9a2d12409078083f596504d6a0d1601594 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 23 Jul 2026 15:08:37 +0200 Subject: [PATCH 37/47] Fix: Check also capitalized dois (e.g., ZENODO) --- tools/scrape-juser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 4632b3c..239f28f 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -440,6 +440,11 @@ class JuserScraper(object): "obo:NCIT_P378": f'https://juser.fz-juelich.de/record/{juser_id}'} if doi in self.pubs.keys(): record = self.pubs[doi] + print('CHECKING EXISTING PUBLICATION WITH DOI ', doi) + # because JUSER is stupid: + elif doi.lower() in self.pubs.keys(): + record = self.pubs[doi.lower()] + print('CHECKING EXISTING PUBLICATION WITH DOI ', doi) else: print('CREATING NEW PUBLICATION FOR DOI ', doi) record = {'schema_type': 'xyzri:XYZPublication', -- 2.52.0 From c4c5ea8d5f5d8b72ccc86f38bedb0af9aae8d50a Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Thu, 23 Jul 2026 15:09:00 +0200 Subject: [PATCH 38/47] clean up debugging residue --- tools/scrape-juser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 239f28f..11d0910 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -468,8 +468,6 @@ class JuserScraper(object): content_type = map_content_to_bibitem.get(kind, None) \ if kind is not None else None kind = content_type - if kind == 'MISC': - import pdb; pdb.set_trace() # add empty attributes for machine prov for data properties # TODO: the slot could also be determined by a different script attributes = record.get('attributes', []) -- 2.52.0 From 13f085972e9cfd862f60041a3119792a80f8cb77 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 10:52:52 +0200 Subject: [PATCH 39/47] remove left-over debug-debris --- tools/scrape-juser.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 11d0910..47f09fb 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -174,14 +174,11 @@ class JuserScraper(object): importedBy, only_self_edits: bool = False, ): - try: - if importedBy.startswith('xyzrins:instruments'): - if not only_self_edits: - return True - else: - return importedBy == self.scriptpid - except AttributeError: - import pdb; pdb.set_trace() + if importedBy.startswith('xyzrins:instruments'): + if not only_self_edits: + return True + else: + return importedBy == self.scriptpid return False def _get_data_property( -- 2.52.0 From bcb71ecb00ab0bc806e74eea86aa7e97f7167329 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 10:53:59 +0200 Subject: [PATCH 40/47] fix: default to string, not None --- tools/scrape-juser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 47f09fb..c99a8b9 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -310,7 +310,7 @@ class JuserScraper(object): if generation.get('object', None) == pof_pid: found = True if self._is_machine_generated( - generation.get('annotations', {}).get('http://purl.org/pav/importedBy', None)): + generation.get('annotations', {}).get('http://purl.org/pav/importedBy', 'nothing')): if p != generation: # overwrite generated_by[i] = p -- 2.52.0 From 36f7423f8a5f384aacac30966ecc57a83feeab1c Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 10:55:00 +0200 Subject: [PATCH 41/47] allow look-ups based on JulID and update person records if new identifiers are found --- tools/scrape-juser.py | 69 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index c99a8b9..9aa9d51 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -203,6 +203,7 @@ class JuserScraper(object): for author in authors: found = False orcid = None + JulID = None # if the author haus an FZJ affiliation, get their info. We can most # reliably look up Orcid based on Julich ID -- external authors # often are only mentioned by name @@ -213,11 +214,18 @@ class JuserScraper(object): logging.info(f'Found author {display_label} under ID {JulID} ' f'with orcid {orcid}: ' f'https://juser.fz-juelich.de/record/{record_id}') + mainid = orcid if orcid is None: - # stop if there is no ORCID for a Juelich-based author + if not JulID: + # stop the author has no orcid and their JulID is also not + # known + continue + # we can make a second attempt with a Julich-issued ID + mainid = self.persons.get(JulID, None) + if not mainid: continue # either get the pid of the preexisting record or build new pid - pid = self.persons.get(orcid, {}).get('pid', + pid = self.persons.get(mainid, {}).get('pid', 'xyzrins:persons/' + str(uuid.uuid4())) person_annotations = \ {'http://purl.org/pav/importedBy': self.scriptpid, @@ -228,7 +236,7 @@ class JuserScraper(object): "object": pid, "annotations": annotations} # submit new person record, if not yet existing - if orcid not in self.persons.keys(): + if mainid not in self.persons.keys(): self._new_person( p_pid=pid, display_label=display_label, @@ -236,6 +244,14 @@ class JuserScraper(object): JulID=JulID, annotations=person_annotations ) + # update the person record, if Juser had any additional infos + else: + self._update_person( + mainid=mainid, + orcid=orcid, + JulID=JulID, + annotations=person_annotations + ) # check if the author is already attributed in the record for i, attribution in enumerate(record.get('attributed_to', [{}])): if pid == attribution.get('object'): @@ -412,10 +428,41 @@ class JuserScraper(object): def _update_person( self, - annotations + mainid, + orcid, + JulID, + annotations, ): - """e.g., add JulID""" - # TODO + """for an existing person record, check ORCID or JulID can be added""" + metadata_JulID = {"schema_type": "dlthings:Identifier", + "creator": "https://w3id.org/isil/DE-Juel1", + "notation": JulID, + "annotations": annotations} + metadata_orcid = {"schema_type": "xyzri:ORCID", + "creator": "ror:04fa4r544", + "notation": orcid, + "annotations": annotations} + person = self.persons.get(mainid) + changes = False + for ident, lookupkey, lookupvalue, meta in \ + [(orcid, 'schema_type', 'xyzri:ORCID', metadata_orcid), + (JulID, 'creator', 'https://w3id.org/isil/DE-Juel1', metadata_JulID)]: + found = False + if ident is None: + continue + for identifier in person.get("identifiers", []): + # what kind of identifier is it? + if identifier.get(lookupkey, None) == lookupvalue: + # found the identifier, no need to add it + found = True + if not found: + print(f"UPDATING PERSON RECORD WITH ID {mainid} WITH {ident}") + person["identifiers"].append(meta) + changes = True + if changes: + # only update record if a change was made + self.persons[mainid] = person + self.to_submit_protected.append(person) return def retrieve_metadata( @@ -604,7 +651,8 @@ def pid_of(x: str | dict) -> str: def process_orcid(person: dict) -> str | None: - """Return an ORCID from identifiers""" + """Return an ORCID from identifiers. If no ORCID is found, try a ZB issued + ID.""" for identifier in person.get("identifiers", []): # this is a deviation from enrich-via-doi, which filtered based on @@ -612,6 +660,13 @@ def process_orcid(person: dict) -> str | None: # fraction of identifiers have this creator annotation if pid_of(identifier.get("schema_type")) == "xyzri:ORCID": return identifier.get("notation") + # if we do not find an ORCID, we make another attempt based on ZB-issued + # ID + for identifier in person.get("identifiers", []): + if pid_of(identifier.get("schema_type")) == "dlthings:Identifier" and \ + pid_of(identifier.get("creator")) == "https://w3id.org/isil/DE-Juel1": + return identifier.get("notation") + map_pof_to_pid = { -- 2.52.0 From ba2ccbdaa85f5a6963fd69533dbbc216fcc64592 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 10:55:36 +0200 Subject: [PATCH 42/47] Fix annotation terms --- tools/scrape-juser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 9aa9d51..4894194 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -480,8 +480,8 @@ class JuserScraper(object): # record of the juser ID, for machine provenance and linking to the # original juser_id = r.fields[0].data - annotations = {"obo:NCIT_C42704": self.scriptpid, - "obo:NCIT_P378": f'https://juser.fz-juelich.de/record/{juser_id}'} + annotations = {"http://purl.org/pav/importedBy": self.scriptpid, + "http://purl.org/pav/importedFrom": f'https://juser.fz-juelich.de/record/{juser_id}'} if doi in self.pubs.keys(): record = self.pubs[doi] print('CHECKING EXISTING PUBLICATION WITH DOI ', doi) -- 2.52.0 From c92f7a1bfd262126ef9187f2d437460c9fa03746 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 10:56:11 +0200 Subject: [PATCH 43/47] only submit updated or new publications --- tools/scrape-juser.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 4894194..ee5eccd 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -483,14 +483,17 @@ class JuserScraper(object): annotations = {"http://purl.org/pav/importedBy": self.scriptpid, "http://purl.org/pav/importedFrom": f'https://juser.fz-juelich.de/record/{juser_id}'} if doi in self.pubs.keys(): - record = self.pubs[doi] + oldrecord = self.pubs[doi] + record = oldrecord print('CHECKING EXISTING PUBLICATION WITH DOI ', doi) # because JUSER is stupid: elif doi.lower() in self.pubs.keys(): - record = self.pubs[doi.lower()] + oldrecord = self.pubs[doi.lower()] + record = oldrecord print('CHECKING EXISTING PUBLICATION WITH DOI ', doi) else: print('CREATING NEW PUBLICATION FOR DOI ', doi) + oldrecord = None record = {'schema_type': 'xyzri:XYZPublication', 'pid': 'xyzrins:publications/' + str(uuid.uuid4()), 'identifiers': [ @@ -542,8 +545,10 @@ class JuserScraper(object): (influenced_by, 'influenced_by')]: if prop: record[slot] = prop - # publication records can be public - self.to_submit_public.append(record) + if oldrecord != record: + # publication records can be public + print('FOUND AN UPDATE FOR ', doi) + self.to_submit_public.append(record) return def _get_doi( -- 2.52.0 From 13e46451d0c61c629174e42199b146c290ca24b5 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 10:56:59 +0200 Subject: [PATCH 44/47] retry journal lookup when xml download is glitchy --- tools/scrape-juser.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index ee5eccd..97afb9d 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -576,10 +576,22 @@ class JuserScraper(object): if peri is None: return None ID = parse.quote_plus(peri) - url = \ - f'https://juser.fz-juelich.de/search?ln=en&cc=Periodicals&p={ID}&f=&action_search=Search&c=Periodicals&c=&sf=&so=d&rm=&rg=10&sc=1&of=xm' - res = self.session.get(url) - x = ET.fromstring(res.content.decode()) + # The retrieval of search results often glitches when ran in short + # succession, so we safe-guard and retry + lookup = False + attempts = 0 + while not lookup and attempts < 3: + try: + url = \ + f'https://juser.fz-juelich.de/search?ln=en&cc=Periodicals&p={ID}&f=&action_search=Search&c=Periodicals&c=&sf=&so=d&rm=&rg=10&sc=1&of=xm' + res = self.session.get(url) + attempts += 1 + x = ET.fromstring(res.content.decode()) + except ET.ParseError as e: + print("JUSER GLITCH, RETRYING...") + continue + finally: + lookup = True # look up ISSN in the XML above. XML is a mess. try: ISSN = x.findall(".//*[@tag='022']")[0][0].text -- 2.52.0 From 2c08f69aaf525ada3699e1b904734d0ab7954a37 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 11:04:15 +0200 Subject: [PATCH 45/47] RF and reuse glitching safeguard --- tools/scrape-juser.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 97afb9d..7635ab8 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -576,22 +576,9 @@ class JuserScraper(object): if peri is None: return None ID = parse.quote_plus(peri) - # The retrieval of search results often glitches when ran in short - # succession, so we safe-guard and retry - lookup = False - attempts = 0 - while not lookup and attempts < 3: - try: - url = \ - f'https://juser.fz-juelich.de/search?ln=en&cc=Periodicals&p={ID}&f=&action_search=Search&c=Periodicals&c=&sf=&so=d&rm=&rg=10&sc=1&of=xm' - res = self.session.get(url) - attempts += 1 - x = ET.fromstring(res.content.decode()) - except ET.ParseError as e: - print("JUSER GLITCH, RETRYING...") - continue - finally: - lookup = True + url = \ + f'https://juser.fz-juelich.de/search?ln=en&cc=Periodicals&p={ID}&f=&action_search=Search&c=Periodicals&c=&sf=&so=d&rm=&rg=10&sc=1&of=xm' + x = self._look_up_juser_info(url, attempts=3) # look up ISSN in the XML above. XML is a mess. try: ISSN = x.findall(".//*[@tag='022']")[0][0].text @@ -599,6 +586,26 @@ class JuserScraper(object): ISSN = None return ISSN + def _look_up_juser_info( + self, + url: str, + attempts: int = 3 + ): + # The retrieval of search results often glitches when ran in short + # succession, so we safe-guard and retry + lookup = False + attempt = 0 + while not lookup and attempt < attempts: + try: + res = self.session.get(url) + attempt += 1 + x = ET.fromstring(res.content.decode()) + except ET.ParseError as e: + print("JUSER GLITCH, RETRYING...") + continue + finally: + lookup = True + return x def _look_up_orcid( self, @@ -611,8 +618,7 @@ class JuserScraper(object): ID = parse.quote_plus(julid) url = \ f'https://juser.fz-juelich.de/search?ln=en&p={ID}&f=&action_search=Search&c=People&sf=author&so=d&rm=&rg=10&sc=0&of=xm' - res = self.session.get(url) - x = ET.fromstring(res.content.decode()) + x = self. _look_up_juser_info(url, attempts=3) # look up ORCID in the XML above. XML is a mess. try: ORCID = \ -- 2.52.0 From 7b66188ba918a341754f1826acd630657b27ba5f Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 13:15:46 +0200 Subject: [PATCH 46/47] fix try except clause --- tools/scrape-juser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/scrape-juser.py b/tools/scrape-juser.py index 7635ab8..b6a6b0c 100644 --- a/tools/scrape-juser.py +++ b/tools/scrape-juser.py @@ -603,8 +603,7 @@ class JuserScraper(object): except ET.ParseError as e: print("JUSER GLITCH, RETRYING...") continue - finally: - lookup = True + lookup = True return x def _look_up_orcid( -- 2.52.0 From 8b2ff59f2097c38f2708246fa444ad773f842b55 Mon Sep 17 00:00:00 2001 From: Adina Wagner Date: Fri, 24 Jul 2026 14:05:36 +0200 Subject: [PATCH 47/47] add brief documentation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 706ab10..5dc3ffa 100644 --- a/README.md +++ b/README.md @@ -11,4 +11,5 @@ with additional machine-generated records. - ``tools/enrich-via-doi.py`` (used by ``.forgejo/workflows/enrich_publications.yml``): reads publication records and extends them with external metadata available via doi.org content negotiation -- ``tools/get-depiction-urls.py`` (used by the 'from-model'-websites, e.g., https://hub.psychoinformatics.de/www/www-from-model/src/branch/main/.forgejo/workflows/register-depictions.yaml): Given metadata input, this script extracts download URLs for each ``Depiction`` of the record, provided the depiction has a ``kind`` included in the depiction-type argument. For each depiction distribution, it will output the record curie, the file extension and the url, to stdout. \ No newline at end of file +- ``tools/get-depiction-urls.py`` (used by the 'from-model'-websites, e.g., https://hub.psychoinformatics.de/www/www-from-model/src/branch/main/.forgejo/workflows/register-depictions.yaml): Given metadata input, this script extracts download URLs for each ``Depiction`` of the record, provided the depiction has a ``kind`` included in the depiction-type argument. For each depiction distribution, it will output the record curie, the file extension and the url, to stdout. +- ``tools/scrape-juser.py`` (used by ``.forgejo/workflows/scrape-juser.yml``): Given an xml export of publications in MARC21 format as provided by Juser (juser.fz-juelich.de), as well as a local cache of XYZPerson and XYZPublication records, this script generates new or updates existing Publication and Person records. Person records are submitted into a 'protected' inbox, whereas Publication records are submitted into a 'public' inbox. \ No newline at end of file -- 2.52.0