47 lines
1.7 KiB
YAML
47 lines
1.7 KiB
YAML
name: Scrape publications from JUSER
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
year:
|
|
description: "Fetch Juser publications from this year (e.g. 2026)"
|
|
required: true
|
|
default: '2026'
|
|
type: string
|
|
|
|
env:
|
|
DTC_TOKEN: ${{ secrets.POOLTOKEN }}
|
|
DUMPTHINGS_APIURL: https://pool.psychoinformatics.de/api
|
|
DUMPTHINGS_COLLECTION: public
|
|
JUSER_PW: ${{ secrets.JUSER_PW }}
|
|
JUSER_USER: ${{ secrets.JUSER_USER }}
|
|
|
|
jobs:
|
|
scrape-juser:
|
|
name: Scrape JUSER
|
|
runs-on: debian-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
steps:
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v6
|
|
- name: Install metadata tools
|
|
run: |
|
|
uv tool install https://hub.psychoinformatics.de/orinoco/query-things.git \
|
|
--with-executables-from dump-things-pyclient
|
|
- name: Fetch script
|
|
run: |
|
|
wget https://hub.psychoinformatics.de/orinoco/knowledge-enrichment/raw/branch/main/tools/scrape-juser.py
|
|
- name: Pre-fetch pool data
|
|
run: |
|
|
mkdir .cache
|
|
dtc get-records $DUMPTHINGS_APIURL public -C XYZPublication > .cache/Publications.jsonl
|
|
dtc get-records $DUMPTHINGS_APIURL public -C XYZPerson > .cache/Person.jsonl
|
|
dtc get-records $DUMPTHINGS_APIURL protected -C XYZPerson >> .cache/Person.jsonl
|
|
- name: Pre-fetch Juser records
|
|
run: |
|
|
curl 'https://juser.fz-juelich.de/PubExporter.py?p=cid%3A%22I%3A%28DE-Juel1%29INM-7-20090406%22+AND+pub%3A%22${{ inputs.year }}%22&sf=author&so=d&rg=&of=xm' > .cache/juser-pubs.xml
|
|
- name: Process records
|
|
run: |
|
|
uv run scrape-juser.py --file .cache/juser-pubs.xml --persons .cache/Person.jsonl --publications .cache/Publications.jsonl
|