From c3ecf7e8a233f5c7596144a01a826abb4a750bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 9 Mar 2026 15:40:28 +0100 Subject: [PATCH 01/10] Copy enrich-via-doi from TRR's pool-publication-page --- .forgejo/tools/enrich-via-doi.py | 392 +++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 .forgejo/tools/enrich-via-doi.py diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py new file mode 100644 index 0000000..ab283ee --- /dev/null +++ b/.forgejo/tools/enrich-via-doi.py @@ -0,0 +1,392 @@ +import json +from urllib.parse import urljoin +from pathlib import Path +import re +import warnings + +import click +from lxml import html +from requests_cache import CachedSession + + +def consult_spdx_license(license_uri: str) -> str | None: + """Match the license uri against spdx data + + This function consults the spdx license file, trying to match the + given url against "reference" or "see also" links. Returns the + label, if match is found. + + """ + spdx_list_data = load_spdx_licenses() + # uri may be lacking "/legalcode" and/or extension, compared to spdx + pat = re.compile(rf"{license_uri}(/legalcode)?(\.[a-z]{{1-5}})?") + + # go through the licenses searching for matching one + res = None + if license_uri.startswith("https://spdx.org"): + # explicit match for "reference" + for license in spdx_list_data["licenses"]: + if re.match(pat, license["reference"]) is not None: + res = license + else: + # match against "see also" urls + for license in spdx_list_data["licenses"]: + for see_also in license["seeAlso"]: + if re.match(pat, see_also) is not None: + res = license + + return res["licenseId"] if res is not None else None + + +def csl_abstract(d: dict) -> str | None: + """Get abstract from csl + + Some abstracts seen in the wild are marked up with jats tags, and + the top-level may include (a combination of) sections, titles and + paragraphs (usually, a section itself contains a title and one + paragraph). We can use the paragraphs, and mix in the section + titles. Otherwise, remove all tags (return text content). + + """ + if abstract := d.get("abstract", False): + h = html.fromstring(abstract) + if {x.tag for x in h} <= {"jats:p", "jats:title", "jats:sec"}: + return jats2md(h) + else: + return h.text_content() + else: + return None + + +def csl_license(d: dict) -> list: + """Get license from doi content-negotiation json""" + license_urls = [] + for license in d.get("license", []): + if license["content-version"] == "vor": + # "version of record" + license_urls.append(license["URL"]) + # deduplicate before returning, just in case + return list(set(license_urls)) + + +def csl_publish_date(d: dict, allow_incomplete: bool = True) -> str | None: + """Get one publication date out of csl""" + if "issued" in d: + date = d["issued"]["date-parts"] + elif "published-online" in d: + date = d["published-online"]["date-parts"] + else: + return None + + # partial date, a nested array of numbers + if len(date[0]) == 1 or (len(date[0]) < 3 and not allow_incomplete): + isodate = f"{date[0][0]}" # yyyy (only year is required) + elif len(date[0]) == 2: + isodate = f"{date[0][0]}-{date[0][1]:02}" # yyyy-mm + else: + isodate = f"{date[0][0]}-{date[0][1]:02}-{date[0][2]:02}" # yyyy-mm-dd + + return isodate + + +def discover_authors( + publication: dict, all_our_people: dict[str, dict], citeproc_record: dict +) -> list[dict]: + + missing_attributions = [] + + # check which contributors with orcids are already declared + declared_contributor_orcids = set() + for attribution in publication.get("attributed_to", []): + if isinstance(attribution, dict): + if (orcid := process_orcid(attribution.get("object", {}))) is not None: + declared_contributor_orcids.add(f"https://orcid.org/{orcid}") + + # compare to contributors with orcids in the citeproc record + for author in citeproc_record.get("author", []): + if ( + (orcid := author.get("ORCID")) is not None + and orcid in all_our_people.keys() + and orcid not in declared_contributor_orcids + ): + if author.get("sequence") == "first": + r = "obo:MS_1002034" # first author + elif author.get("sequence") == "additional": + r = "obo:MS_1002036" # co-author + else: + r = "marcrel:aut" + missing_attributions.append({"object": all_our_people[orcid], "roles": [r]}) + + return missing_attributions + + +def jats2md(span: html.HtmlElement, rstrip: bool = True) -> str: + full_text = "" + for elem in span: + if elem.tag == "jats:title": + if elem.text.lower() != "abstract": + # we know an abstract is an abstract + full_text += elem.text_content() + full_text += ": " if not elem.text_content().endswith(".") else " " + elif elem.tag == "jats:p": + this_text = elem.text_content() + for sub in elem: + if sub.tag == "jats:ext-link": + # wrap at least plain links for unambiguous parsing by hugo + if (href := sub.get("xlink:href")) == sub.text_content(): + this_text = this_text.replace(href, f"<{href}>") + full_text += this_text + full_text += "\n\n" + elif elem.tag == "jats:sec": + full_text += jats2md(elem, rstrip=False) + else: + full_text += elem.text_content() + return full_text.rstrip() if rstrip else full_text + + +def load_spdx_licenses(lic_file: Path = Path(".cache/licenses.json")) -> dict: + """Load spdx license file - from Internet or disk + + If loading from Internet, store in a file for future use. + + """ + if lic_file.exists(): + with lic_file.open() as f: + d = json.load(f) + else: + # "permanently" cache by downloading + with CachedSession(backend="memory") as session: + r = session.get("https://spdx.org/licenses/licenses.json") + if r.ok: + d = r.json() + with lic_file.open("w") as f: + json.dump(d, f) + else: + warnings.warn("Failed to retrieve the spdx license file") + d = {"licenses": []} + return d + + +def process_doi(paper: dict) -> str | None: + """Return a DOI from identifiers""" + + # TODO: use inlined form + for identifier in paper.get("identifiers", []): + if ( + identifier.get("creator") == "ror:01fyxcz70" + or identifier.get("schema_type") == "dlthings:DOI" + ): + return identifier.get("notation") + + +def process_orcid(person: dict) -> str | None: + """Return an ORCID from identifiers""" + + # TODO: use inlined form + for identifier in person.get("identifiers", []): + if ( + identifier.get("schema_type") == "trr379ri:ORCID" + or identifier.get("creator") == "ror:04fa4r544" + ): + return identifier.get("notation") + + +def publishing_process(d: dict) -> dict[str, str] | None: + res = {"object": "obo:IAO_0000444"} + has_detail = False + + if (pubdate := csl_publish_date(d)) is not None: + has_detail = True + res["at_time"] = pubdate + + if (issn := d["ISSN"]) is not None: + has_detail = True + # there can be more than one (e.g. different for print / online) + # if that's the case, use the 1st - we have no more data at hand + res["at_location"] = f"ISSN:{issn[0]}" + + return res if has_detail else None + + +def query_doi_citation(session: CachedSession, doi: str) -> str | None: + doi_url = urljoin("https://doi.org/", doi) + r = session.get(doi_url, headers={"Accept": "text/x-bibliography; style=apa"}) + if r.ok and (r.encoding != r.apparent_encoding == "utf-8"): + # if it appears like utf-8, it likely is utf-8 + # see https://stackoverflow.com/questions/44203397/ + r.encoding = r.apparent_encoding + return r.text if r.ok else None + + +def query_doi_csl(session: CachedSession, doi: str) -> dict | None: + doi_url = urljoin("https://doi.org", doi) + r = session.get( + doi_url, headers={"Accept": "application/vnd.citationstyles.csl+json"} + ) + return r.json() if r.ok else None + + +def remap_person_records(records: list[dict]) -> dict[str, dict]: + orcid_map = { + f"https://orcid.org/{orcid}": record + for record in records + if (orcid := process_orcid(record)) is not None + } + return orcid_map + + +def rules(citeproc_record: dict) -> list[str]: + res = [] + for url in csl_license(citeproc_record): + if (license_label := consult_spdx_license(url)) is not None: + res.append(f"spdxlic:{license_label}") + return sorted(res) + + +def short_name_from_citeproc(d: dict) -> str | None: + """Generate file name based on citeproc data + + Combines last name of the first author, (short) container title, + and date to form something that is human-readable and likely + unique enough. + + Required properties are usually present, but they are not + required, so we proceed only if we find all three. + + """ + + if not ( + "author" in d + and ("container-title-short" in d or "container-title" in d) + and "issued" in d + ): + return None + + # first author (et al) + author = d["author"] + if len(author) == 1: + # family is required (at least in crossref) - define default to be safe + author_part = author[0].get("family", "unknown") + else: + author_part = author[0].get("family", "unknown") + "_etal" + + # journal title (abbreviated) + if container := d.get("container-title-short", False): + journal_part = container.replace(" ", "_") + elif ((container := d.get("container-title")) is not None) and container != []: + # todo: iso4? + journal_part = container.replace(" ", "_") + else: + # none of those are mandatory + journal_part = d.get("group-title", "") + institution = d.get("institution", [{}])[0].get("name") + if institution == "bioRxiv": + # "biorxiv-neuroscience" over "neuroscience" + journal_part = institution + "-" + "journal_part" + if journal_part == "": + journal_part = "unknown" + journal_part = re.sub(r"[^\w]", "", journal_part) # keep alphanumerics + + date_part = csl_publish_date(d).replace("-", "_") # pyright:ignore + + return "_".join((author_part, journal_part, date_part)) + ".md" + + +@click.command() +@click.argument("input", type=click.File("rb")) +@click.argument("persons", type=click.File("rb")) +@click.argument("output", type=click.File("wt")) +@click.option("--extras", is_flag=True) +def main(input, persons, output, extras): + + session = CachedSession( + ".cache/requests-cache/http_cache", + backend="sqlite", + match_headers=["Accept"], + expire_after=7200, + ) + + all_people = [json.loads(line) for line in persons] + all_people_dict = remap_person_records(all_people) + + for line in input: + paper = json.loads(line) + doi = process_doi(paper) + citeproc_metadata = query_doi_csl(session, doi) if doi is not None else None + citation_text = ( + query_doi_citation(session, doi) if doi is not None and extras else None + ) + + if citation_text is not None: + paper["x_citation"] = citation_text + + if citeproc_metadata is None: + # nothing to do, emit unchanged + click.echo(json.dumps(paper), output) + continue + + # contributors + more_attributions = discover_authors(paper, all_people_dict, citeproc_metadata) + if len(more_attributions) > 0: + if "attributed_to" not in paper: + paper["attributed_to"] = more_attributions + else: + paper["attributed_to"].extend(more_attributions) + + # publishing activity (date / ISSN) + citeproc_pp = publishing_process(citeproc_metadata) + activities = paper.get("generated_by", []) + + # find publishing process in publication + pp_idx = None + for i in range(len(activities)): + # TODO: also deal with inlined form + if activities[i].get("object") == "obo:IAO_0000444": # Publishing process + pp_idx = i + break + + # update publishing activity (date & issn) + if citeproc_pp is not None: + if "generated_by" not in paper: + # no activities so far: add a list + paper["generated_by"] = [citeproc_pp] + elif pp_idx is None: + # activities but no publishing process: append + paper["generated_by"].append(citeproc_pp) + else: + # activities incl. publishing process: merge keeping original values + paper["generated_by"][pp_idx] = ( + citeproc_pp | paper["generated_by"][pp_idx] + ) + # override date if is more precise in citeproc + if len(citeproc_pp.get("at_time", "").split("-")) > len( + paper["generated_by"][pp_idx].get("at_time", "").split("-") + ): + paper["generated_by"][pp_idx]["at_time"] = citeproc_pp["at_time"] + + # title + if paper.get("title") is None and citeproc_metadata.get("title") is not None: + paper["title"] = citeproc_metadata.get("title") + + # abstract + if ( + paper.get("description") is None + and (citeproc_abstract := csl_abstract(citeproc_metadata)) is not None + ): + paper["description"] = citeproc_abstract + + # rules (licenses) + if paper.get("rules") is None: + citeproc_rules = rules(citeproc_metadata) + if len(citeproc_rules) > 0: + paper["rules"] = citeproc_rules + + # suggested output file name + if extras and (sn := short_name_from_citeproc(citeproc_metadata)) is not None: + paper["x_suggested_name"] = sn + + click.echo(json.dumps(paper), output) + + +if __name__ == "__main__": + main() -- 2.52.0 From a4916305ca069783f2b5ece96b794981848663b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 9 Mar 2026 16:42:51 +0100 Subject: [PATCH 02/10] Tweak doi enrichment Remove TRR prefix, temporarily disable cache, be flexible for inlined/pid-only, add help, change the regular script to uv script. --- .forgejo/tools/enrich-via-doi.py | 68 ++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py index ab283ee..447b80b 100644 --- a/.forgejo/tools/enrich-via-doi.py +++ b/.forgejo/tools/enrich-via-doi.py @@ -1,10 +1,19 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "rich-click", +# "lxml", +# "requests_cache", +# ] +# /// + import json from urllib.parse import urljoin from pathlib import Path import re import warnings -import click +import rich_click as click from lxml import html from requests_cache import CachedSession @@ -155,7 +164,8 @@ def load_spdx_licenses(lic_file: Path = Path(".cache/licenses.json")) -> dict: d = json.load(f) else: # "permanently" cache by downloading - with CachedSession(backend="memory") as session: + session = CachedSession(backend="memory") + with session.cache_disabled(): r = session.get("https://spdx.org/licenses/licenses.json") if r.ok: d = r.json() @@ -167,13 +177,23 @@ def load_spdx_licenses(lic_file: Path = Path(".cache/licenses.json")) -> dict: return d +def pid_of(x: str | dict) -> str: + """Return a PID of an object, inlined or not + + A shortcut - makes a pid string or an inlined dict (where pid is a + property) equivalent. Does not do further validation, but it could + be added here. + + """ + return x.get("pid", "") if isinstance(x, dict) else x + + def process_doi(paper: dict) -> str | None: """Return a DOI from identifiers""" - # TODO: use inlined form for identifier in paper.get("identifiers", []): if ( - identifier.get("creator") == "ror:01fyxcz70" + pid_of(identifier.get("creator")) == "ror:01fyxcz70" or identifier.get("schema_type") == "dlthings:DOI" ): return identifier.get("notation") @@ -182,12 +202,8 @@ def process_doi(paper: dict) -> str | None: def process_orcid(person: dict) -> str | None: """Return an ORCID from identifiers""" - # TODO: use inlined form for identifier in person.get("identifiers", []): - if ( - identifier.get("schema_type") == "trr379ri:ORCID" - or identifier.get("creator") == "ror:04fa4r544" - ): + if pid_of(identifier.get("creator")) == "ror:04fa4r544": return identifier.get("notation") @@ -296,8 +312,37 @@ def short_name_from_citeproc(d: dict) -> str | None: @click.argument("input", type=click.File("rb")) @click.argument("persons", type=click.File("rb")) @click.argument("output", type=click.File("wt")) -@click.option("--extras", is_flag=True) +@click.option("--extras", is_flag=True, help="Add non-schema-compliant properties (starting with x_).") def main(input, persons, output, extras): + """Enrich record with metadata fetched via doi.org + + Reads publication records from INPUT, person records from PERSONS, + and outputs enriched records to OUTPUT. INPUT, PERSONS, and OUTPUT + should be in JSON lines format, and can be files or stdin / stdout + (-). + + Authors in the retrieved metadata will be cross-referenced with + the available Person records based on ORCID, and added to + contributors (requires ORCID to be present in both + sources). Licenses will be translated to use SPDX identifiers as + PIDs (e.g. from creative commons canonical URLs) if the license + URL is available in the SPDX database. + + Only the properties which are missing are updated (date is the + exception, updated if more precise one is available). + + If --extras is specified, the produced record will contain + properties which are not compatible with the research information + schema, but can be useful for page generators (x_citation and + x_suggested_name). + + Makes requests to doi.org (content negotiation) to fetch metadata + (and, with --extras, also formatted citation). Also retrieves SPDX + license file (to reference licenses). Uses caching to store + requests in `$PWD/.cache` (doi.org valid for 2 hours, spdx file + until removed). + + """ session = CachedSession( ".cache/requests-cache/http_cache", @@ -340,8 +385,7 @@ def main(input, persons, output, extras): # find publishing process in publication pp_idx = None for i in range(len(activities)): - # TODO: also deal with inlined form - if activities[i].get("object") == "obo:IAO_0000444": # Publishing process + if pid_of(activities[i].get("object")) == "obo:IAO_0000444": # Publishing process pp_idx = i break -- 2.52.0 From 7972ff42132b08f27d51bdec498ffb7974bfe03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 9 Mar 2026 18:20:14 +0100 Subject: [PATCH 03/10] Add only PID when enriching publication with attributions Because of the origins as part of a page generator, the Person enrichment added entire Person records. API submission only needs PIDs. This one-line change adapts the script to use in API submission. Page generators can inline the records if needed. --- .forgejo/tools/enrich-via-doi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py index 447b80b..37e1473 100644 --- a/.forgejo/tools/enrich-via-doi.py +++ b/.forgejo/tools/enrich-via-doi.py @@ -124,7 +124,7 @@ def discover_authors( r = "obo:MS_1002036" # co-author else: r = "marcrel:aut" - missing_attributions.append({"object": all_our_people[orcid], "roles": [r]}) + missing_attributions.append({"object": all_our_people[orcid]["pid"], "roles": [r]}) return missing_attributions -- 2.52.0 From f83aee9da4273bc3c7dfbbdbab56fce36c6c62b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Fri, 13 Mar 2026 18:57:36 +0100 Subject: [PATCH 04/10] Switch DOI enrichment to work on PIDs, not inlined records Using the script for record enrichment (ie. feeding data back into the pool) means that the record we produce should not contain inlined items. We still need access to all known person records (to match external metadata with existing records) but in those we are only interested in the ORCID IDs. So the input (publication) records do not require prior inlining of attribution objects. What we really need is a bidirectional mapping: from PID to ORCID (to know which contributors are already credited) and from ORCID to PID (to add more contributors). This functionality is conveniently provided by bidict, which is an external dependency but it is a tiny one (33 kB wheel). This change allows the code to work with records containing attributions in which objects are not inlined. In the current form, we lose the ability to work with inlined records (this can be brought back by looking up record's pid) but in the enrichment context we don't need that, and not inlining is leaner. In this context, any rendering (e.g. website) would probably use the record after it has been submitted back into the pool. --- .forgejo/tools/enrich-via-doi.py | 35 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py index 37e1473..3d3b892 100644 --- a/.forgejo/tools/enrich-via-doi.py +++ b/.forgejo/tools/enrich-via-doi.py @@ -1,6 +1,7 @@ # /// script # requires-python = ">=3.12" # dependencies = [ +# "bidict", # "rich-click", # "lxml", # "requests_cache", @@ -13,9 +14,10 @@ from pathlib import Path import re import warnings -import rich_click as click +from bidict import bidict from lxml import html from requests_cache import CachedSession +import rich_click as click def consult_spdx_license(license_uri: str) -> str | None: @@ -99,7 +101,7 @@ def csl_publish_date(d: dict, allow_incomplete: bool = True) -> str | None: def discover_authors( - publication: dict, all_our_people: dict[str, dict], citeproc_record: dict + publication: dict, known_people: bidict[str, str], citeproc_record: dict ) -> list[dict]: missing_attributions = [] @@ -107,15 +109,14 @@ def discover_authors( # check which contributors with orcids are already declared declared_contributor_orcids = set() for attribution in publication.get("attributed_to", []): - if isinstance(attribution, dict): - if (orcid := process_orcid(attribution.get("object", {}))) is not None: - declared_contributor_orcids.add(f"https://orcid.org/{orcid}") + if (orcid := known_people.get(attribution.get("object"))) is not None: + declared_contributor_orcids.add(orcid) # compare to contributors with orcids in the citeproc record for author in citeproc_record.get("author", []): if ( (orcid := author.get("ORCID")) is not None - and orcid in all_our_people.keys() + and orcid in known_people.values() and orcid not in declared_contributor_orcids ): if author.get("sequence") == "first": @@ -124,7 +125,9 @@ def discover_authors( r = "obo:MS_1002036" # co-author else: r = "marcrel:aut" - missing_attributions.append({"object": all_our_people[orcid]["pid"], "roles": [r]}) + missing_attributions.append( + {"object": known_people.inverse[orcid], "roles": [r]} + ) return missing_attributions @@ -242,13 +245,13 @@ def query_doi_csl(session: CachedSession, doi: str) -> dict | None: return r.json() if r.ok else None -def remap_person_records(records: list[dict]) -> dict[str, dict]: - orcid_map = { - f"https://orcid.org/{orcid}": record - for record in records - if (orcid := process_orcid(record)) is not None - } - return orcid_map +def remap_person_records(records: list[dict]) -> bidict[str, str]: + """Create a bidirectional mapping of PIDs and ORCIDs""" + my_map = bidict() + for record in records: + if (orcid := process_orcid(record)) is not None: + my_map[record["pid"]] = f"https://orcid.org/{orcid}" + return my_map def rules(citeproc_record: dict) -> list[str]: @@ -352,7 +355,7 @@ def main(input, persons, output, extras): ) all_people = [json.loads(line) for line in persons] - all_people_dict = remap_person_records(all_people) + pid_orcid_map = remap_person_records(all_people) for line in input: paper = json.loads(line) @@ -371,7 +374,7 @@ def main(input, persons, output, extras): continue # contributors - more_attributions = discover_authors(paper, all_people_dict, citeproc_metadata) + more_attributions = discover_authors(paper, pid_orcid_map, citeproc_metadata) if len(more_attributions) > 0: if "attributed_to" not in paper: paper["attributed_to"] = more_attributions -- 2.52.0 From b7a6f70114b00c4d2c87f9b64ca38169c2a9600c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 16 Mar 2026 13:46:01 +0100 Subject: [PATCH 05/10] Use Rule records, not SPDX, for license discovery Because a Rule record can declare exact mappings, we will use them instead of an ad-hoc downloaded spdx file. This makes the process more self-contained. This means that there is more reliance on the information maintained in the pool (vs. reliance on the use of external identifiers and information available through them from elsewhere) but in the case of mapping license identifiers (in practice, between spdx and creative commons namespaces) this seems to be in line with the spirit of things. One thing I wasn't quite sure about are trailing "/" on the identifiers ("canonical URLs" for creative commons do have them) and whether they should be allowed / expected in the exact mappings. The comparisons for exact mappings are done with the trailing "/" stripped to be on the safe side. --- .forgejo/tools/enrich-via-doi.py | 101 +++++++++++++------------------ 1 file changed, 43 insertions(+), 58 deletions(-) diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py index 3d3b892..fe0b599 100644 --- a/.forgejo/tools/enrich-via-doi.py +++ b/.forgejo/tools/enrich-via-doi.py @@ -20,33 +20,29 @@ from requests_cache import CachedSession import rich_click as click -def consult_spdx_license(license_uri: str) -> str | None: - """Match the license uri against spdx data +def consult_rules(license_uri: str, rules: list[dict]) -> str | None: + """Match a license URI against Rule records - This function consults the spdx license file, trying to match the - given url against "reference" or "see also" links. Returns the - label, if match is found. + This function tries to match the given license url against PIDs + (expanded from curie to uri using hardcoded prefixes) or exact + mappings. """ - spdx_list_data = load_spdx_licenses() - # uri may be lacking "/legalcode" and/or extension, compared to spdx - pat = re.compile(rf"{license_uri}(/legalcode)?(\.[a-z]{{1-5}})?") - # go through the licenses searching for matching one - res = None - if license_uri.startswith("https://spdx.org"): - # explicit match for "reference" - for license in spdx_list_data["licenses"]: - if re.match(pat, license["reference"]) is not None: - res = license - else: - # match against "see also" urls - for license in spdx_list_data["licenses"]: - for see_also in license["seeAlso"]: - if re.match(pat, see_also) is not None: - res = license + pmap = { + "obo": "http://purl.obolibrary.org/obo/", + "spdxlic": "https://spdx.org/licenses/", + } - return res["licenseId"] if res is not None else None + for rule in rules: + # this assumes pid needs expansion but exact mappings do not + # this ignores trailing / in exact mappings + identifiers = [ + expand_curie(rule["pid"], pmap), + *[x.rstrip("/") for x in rule.get("exact_mappings", [])], + ] + if license_uri.rstrip("/") in identifiers: + return rule["pid"] def csl_abstract(d: dict) -> str | None: @@ -132,6 +128,20 @@ def discover_authors( return missing_attributions +def expand_curie(curie: str, pmap: dict[str, str]) -> str: + """Expand curie to uri using a prefix map + + If there is no prefix or the prefix is not defined in the prefix + map, returns the input value. This is a simple helper. For more + complex usecases, consider using the external curies package. + + """ + pat = re.compile(r"(?P\w+):(?P.*)") + if (m := re.match(pat, curie)) is not None and m["prefix"] in pmap: + return pmap[m["prefix"]] + m["reference"] + return curie + + def jats2md(span: html.HtmlElement, rstrip: bool = True) -> str: full_text = "" for elem in span: @@ -156,30 +166,6 @@ def jats2md(span: html.HtmlElement, rstrip: bool = True) -> str: return full_text.rstrip() if rstrip else full_text -def load_spdx_licenses(lic_file: Path = Path(".cache/licenses.json")) -> dict: - """Load spdx license file - from Internet or disk - - If loading from Internet, store in a file for future use. - - """ - if lic_file.exists(): - with lic_file.open() as f: - d = json.load(f) - else: - # "permanently" cache by downloading - session = CachedSession(backend="memory") - with session.cache_disabled(): - r = session.get("https://spdx.org/licenses/licenses.json") - if r.ok: - d = r.json() - with lic_file.open("w") as f: - json.dump(d, f) - else: - warnings.warn("Failed to retrieve the spdx license file") - d = {"licenses": []} - return d - - def pid_of(x: str | dict) -> str: """Return a PID of an object, inlined or not @@ -254,11 +240,11 @@ def remap_person_records(records: list[dict]) -> bidict[str, str]: return my_map -def rules(citeproc_record: dict) -> list[str]: +def rules_from_citeproc(citeproc_record: dict, known_rules: list[dict]) -> list[str]: res = [] for url in csl_license(citeproc_record): - if (license_label := consult_spdx_license(url)) is not None: - res.append(f"spdxlic:{license_label}") + if (license_pid := consult_rules(url, known_rules)) is not None: + res.append(license_pid) return sorted(res) @@ -315,8 +301,9 @@ def short_name_from_citeproc(d: dict) -> str | None: @click.argument("input", type=click.File("rb")) @click.argument("persons", type=click.File("rb")) @click.argument("output", type=click.File("wt")) +@click.option("--rules", type=click.File("rb"), help="Rule records (jsonl file) to reference.") @click.option("--extras", is_flag=True, help="Add non-schema-compliant properties (starting with x_).") -def main(input, persons, output, extras): +def main(input, persons, output, rules, extras): """Enrich record with metadata fetched via doi.org Reads publication records from INPUT, person records from PERSONS, @@ -327,9 +314,8 @@ def main(input, persons, output, extras): Authors in the retrieved metadata will be cross-referenced with the available Person records based on ORCID, and added to contributors (requires ORCID to be present in both - sources). Licenses will be translated to use SPDX identifiers as - PIDs (e.g. from creative commons canonical URLs) if the license - URL is available in the SPDX database. + sources). Licenses will be translated by checking PIDs and exact + mappings of the provided Rule records. Only the properties which are missing are updated (date is the exception, updated if more precise one is available). @@ -340,10 +326,8 @@ def main(input, persons, output, extras): x_suggested_name). Makes requests to doi.org (content negotiation) to fetch metadata - (and, with --extras, also formatted citation). Also retrieves SPDX - license file (to reference licenses). Uses caching to store - requests in `$PWD/.cache` (doi.org valid for 2 hours, spdx file - until removed). + (and, with --extras, also formatted citation). Uses caching to + store requests in `$PWD/.cache` (valid for 2 hours). """ @@ -355,6 +339,7 @@ def main(input, persons, output, extras): ) all_people = [json.loads(line) for line in persons] + all_rules = [json.loads(line) for line in rules] if rules is not None else [] pid_orcid_map = remap_person_records(all_people) for line in input: @@ -424,7 +409,7 @@ def main(input, persons, output, extras): # rules (licenses) if paper.get("rules") is None: - citeproc_rules = rules(citeproc_metadata) + citeproc_rules = rules_from_citeproc(citeproc_metadata, all_rules) if len(citeproc_rules) > 0: paper["rules"] = citeproc_rules -- 2.52.0 From 1c4b1f4fe99f215e355bc1bf26d87ccbeb112bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Mon, 16 Mar 2026 17:10:17 +0100 Subject: [PATCH 06/10] Make person records optional for DOI enrichment This changes the CLI to only have INPUT and OUTPUT as arguments; additional records (Person and Rule) now need to be provided as options. If not provided, respective part of enrichmant won't be performed. With neither Person nor Rule, enrichment can still add date, ISSN, title, and abstract. --- .forgejo/tools/enrich-via-doi.py | 42 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py index fe0b599..ed0b629 100644 --- a/.forgejo/tools/enrich-via-doi.py +++ b/.forgejo/tools/enrich-via-doi.py @@ -299,23 +299,35 @@ def short_name_from_citeproc(d: dict) -> str | None: @click.command() @click.argument("input", type=click.File("rb")) -@click.argument("persons", type=click.File("rb")) @click.argument("output", type=click.File("wt")) -@click.option("--rules", type=click.File("rb"), help="Rule records (jsonl file) to reference.") -@click.option("--extras", is_flag=True, help="Add non-schema-compliant properties (starting with x_).") -def main(input, persons, output, rules, extras): +@click.option( + "--persons", + type=click.File("rb"), + help="Person records to discover authors (json lines).", +) +@click.option( + "--rules", + type=click.File("rb"), + help="Rule records (json lines) to match licenses (json lines).", +) +@click.option( + "--extras", + is_flag=True, + help="Add non-schema-compliant properties (starting with x_).", +) +def main(input, output, persons, rules, extras): """Enrich record with metadata fetched via doi.org - Reads publication records from INPUT, person records from PERSONS, - and outputs enriched records to OUTPUT. INPUT, PERSONS, and OUTPUT - should be in JSON lines format, and can be files or stdin / stdout - (-). - - Authors in the retrieved metadata will be cross-referenced with - the available Person records based on ORCID, and added to - contributors (requires ORCID to be present in both - sources). Licenses will be translated by checking PIDs and exact - mappings of the provided Rule records. + Reads publication records from INPUT and outputs enriched records + to OUTPUT. INPUT and OUTPUT should be in JSON lines format, and + can be files or stdin / stdout (-). + + With --persons, authors in the retrieved metadata will be + cross-referenced with the provided Person records based on ORCID, + and added to contributors (requires ORCID to be present in both + sources). With --rules, licenses will be translated by checking + PIDs and exact mappings of the provided Rule records. Both + arguments can use JSON lines files or stdin (-). Only the properties which are missing are updated (date is the exception, updated if more precise one is available). @@ -338,7 +350,7 @@ def main(input, persons, output, rules, extras): expire_after=7200, ) - all_people = [json.loads(line) for line in persons] + all_people = [json.loads(line) for line in persons] if persons is not None else [] all_rules = [json.loads(line) for line in rules] if rules is not None else [] pid_orcid_map = remap_person_records(all_people) -- 2.52.0 From 8a8c7bda20177ab5b781602fd96e88ced484639b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Tue, 17 Mar 2026 19:48:12 +0100 Subject: [PATCH 07/10] Mention DOI enrichment in the README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c8996d7..5484ddc 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,5 @@ with additional machine-generated records. - ``.forgejo/tools/scrape-calendar.py`` (used by ``.forgejo/workflows/scrape.yml``): scrapes three FZJ funding calendars and adds their events as XYZCompetition records (research information scheme) +- ``.forgejo/tools/enrich-via-doi.py``: reads publication records and extends + them with external metadata available via doi.org content negotiation -- 2.52.0 From 4dda0f3c8b219540580a2d31d2c51bbc075ca4f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Wed, 25 Mar 2026 15:06:22 +0100 Subject: [PATCH 08/10] Treat ISSN in csl+json metadata as optional --- .forgejo/tools/enrich-via-doi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/tools/enrich-via-doi.py b/.forgejo/tools/enrich-via-doi.py index ed0b629..c3d7e1b 100644 --- a/.forgejo/tools/enrich-via-doi.py +++ b/.forgejo/tools/enrich-via-doi.py @@ -204,7 +204,7 @@ def publishing_process(d: dict) -> dict[str, str] | None: has_detail = True res["at_time"] = pubdate - if (issn := d["ISSN"]) is not None: + if (issn := d.get("ISSN")) is not None: has_detail = True # there can be more than one (e.g. different for print / online) # if that's the case, use the 1st - we have no more data at hand -- 2.52.0 From d340ec508c608ee122447d63ee8c688c315621bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Wed, 25 Mar 2026 15:05:19 +0100 Subject: [PATCH 09/10] Add enrich publications workflow This adds a workflow which runs the publication enrichment via doi.org. Given that the DOI org information will change very rarely, and we don't (yet) have ways to say "this record is complete / needs no enrichment", the workflow currently only has a "workflow dispatch" trigger. Two optional inputs can be specified when dispatching the workflow: list of PIDs and inbox label. These will limit processing to a subset of records. Otherwise, all records will be processed. Properties which can change based on the pool / data model (API URL, collection name, class names) are kept as env variables to make tweaks easier. In the last step (process record), inputs are assigned (export) to environment variables to avoid issues when the runner is filling them in (eg. end of line after `<<<` when pids are not provided was a syntax error). To supply the optional `--incoming label` argument to dtc get-records, parameter expansion is used (`${parameter:+word}` expands to nothing if parameter is null or unset, otherwise expansion of word is used). --- .forgejo/workflows/enrich_publications.yml | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .forgejo/workflows/enrich_publications.yml diff --git a/.forgejo/workflows/enrich_publications.yml b/.forgejo/workflows/enrich_publications.yml new file mode 100644 index 0000000..95bc68b --- /dev/null +++ b/.forgejo/workflows/enrich_publications.yml @@ -0,0 +1,64 @@ +name: Enrich publications via doi.org + +on: + workflow_dispatch: + inputs: + pids: + description: "Limit to these PIDs (comma-separated)" + required: false + default: '' + type: string + inbox: + description: "Limit to inbox with this label" + required: false + default: '' + type: string + +env: + DTC_TOKEN: ${{ secrets.POOLTOKEN }} + DUMPTHINGS_APIURL: https://pool.psychoinformatics.de/api + DUMPTHINGS_COLLECTION: public + PERSON_CLASS: XYZPerson + PUBLICATION_CLASS: XYZPublication + RULE_CLASS: Rule + +jobs: + enrich-publications: + name: Enrich publications + runs-on: debian-latest + defaults: + run: + shell: bash + steps: + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Install metadata tools + run: | + uv tool install https://hub.psychoinformatics.de/orinoco/query-things.git \ + --with-executables-from dump-things-pyclient + - name: Fetch script + run: | + wget https://hub.psychoinformatics.de/orinoco/knowledge-enrichment/raw/branch/main/.forgejo/tools/enrich-via-doi.py + - name: Pre-fetch data + run: | + mkdir .cache + dtc get-records $DUMPTHINGS_APIURL public -C $PERSON_CLASS > .cache/Person.jsonl + dtc get-records $DUMPTHINGS_APIURL public -C $RULE_CLASS > .cache/Rule.jsonl + - name: Process records + run: | + export INBOX_LABEL=${{ inputs.inbox }} + export PIDS=${{ inputs.pids }} + if [ -n "$PIDS" ] + then + IFS=',' read -ra PID_ARRAY <<< $PIDS + for pid in ${PID_ARRAY[@]} + do + dtc get-records $DUMPTHINGS_APIURL $DUMPTHINGS_COLLECTION --pid $pid ${INBOX_LABEL:+--incoming $INBOX_LABEL} | + uv run enrich-via-doi.py --persons .cache/Person.jsonl --rules .cache/Rule.jsonl - - | + dtc post-records $DUMPTHINGS_APIURL $DUMPTHINGS_COLLECTION $PUBLICATION_CLASS + done + else + dtc get-records $DUMPTHINGS_APIURL $DUMPTHINGS_COLLECTION --class $PUBLICATION_CLASS ${INBOX_LABEL:+--incoming $INBOX_LABEL} | + uv run enrich-via-doi.py --persons .cache/Person.jsonl --rules .cache/Rule.jsonl - - | + dtc post-records $DUMPTHINGS_APIURL $DUMPTHINGS_COLLECTION $PUBLICATION_CLASS + fi -- 2.52.0 From 37b0d1d30dc86700c24cc6edff668c0dda7eb86d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szczepanik?= Date: Wed, 25 Mar 2026 16:49:19 +0100 Subject: [PATCH 10/10] Mention the workflow in the readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5484ddc..bb78ae9 100644 --- a/README.md +++ b/README.md @@ -8,5 +8,6 @@ with additional machine-generated records. - ``.forgejo/tools/scrape-calendar.py`` (used by ``.forgejo/workflows/scrape.yml``): scrapes three FZJ funding calendars and adds their events as XYZCompetition records (research information scheme) -- ``.forgejo/tools/enrich-via-doi.py``: reads publication records and extends - them with external metadata available via doi.org content negotiation +- ``.forgejo/tools/enrich-via-doi.py`` (used by + ``.forgejo/workflows/enrich_publications.yml``): reads publication records and + extends them with external metadata available via doi.org content negotiation -- 2.52.0