From cc620d69ac74d0bab3ab69a8dbec63a75acb72ab Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Wed, 26 Aug 2026 09:29:46 +0200 Subject: [PATCH] add(models): add AT model --- .../rdm/records/transform/models/_config.py | 4 ++ .../rdm/records/transform/models/at.py | 56 +++++++++++++++++++ .../transform/xml_processing/rules/at.py | 42 ++++++++++++++ .../transform/xml_processing/rules/base.py | 17 +++++- .../xml_processing/rules/research.py | 30 +++++----- cds_migrator_kit/rdm/streams.yaml | 17 +++++- setup.cfg | 6 ++ 7 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/models/at.py create mode 100644 cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py diff --git a/cds_migrator_kit/rdm/records/transform/models/_config.py b/cds_migrator_kit/rdm/records/transform/models/_config.py index 225b81fa..5a557e7d 100644 --- a/cds_migrator_kit/rdm/records/transform/models/_config.py +++ b/cds_migrator_kit/rdm/records/transform/models/_config.py @@ -1,10 +1,12 @@ """Fields which we can confidently ignore in each model.""" + IGNORE_SYSTEM_KEYS = { "0248_a", "0248_p", "0248_q", "852__c", # holdings will be taken separately "852__h", + "035__z", # arxiv identifier (e.g. https://cds.cern.ch/record/1054291/export/hm?no_redirect_migrated) "037__c", # arxiv subject "100__m", # email of contributor "300__a", # number of pages @@ -13,6 +15,8 @@ "8564_s", # bibdoc id "8564_x", # icon thumbnails sizes "8564_y", # file description - done by files dump + "8564_8", # File information (done by file dump) + "8564_q", # File links File information (done by file dump) "916__y", # year, redundant value "937__c", # last modified by "937__s", # last modification date diff --git a/cds_migrator_kit/rdm/records/transform/models/at.py b/cds_migrator_kit/rdm/records/transform/models/at.py new file mode 100644 index 00000000..60756cc5 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/at.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) +from cds_migrator_kit.transform.overdo import CdsOverdo + + +class ATModel(CdsOverdo): + """Translation model for AT records.""" + + __query__ = """ + (980__:ARTICLE OR 980__:PREPRINT) + AND + (710__.5:NPA OR 710__.5:AT) + -710__.5:SI -710__.5:SC -710__.5:SL -710__.5:PS -710__.5:MPS -710__.5:ISR -710__.5:MSC -710__.5:AC -710__.5:SPS + -710__.5:LEP -710__.5:AB -710__.5:AR -710__.5:TS -710__.5:ST -710__.5:MT -710__.5:EST -710__.5:SB -710__.5:LHC + -980__:DELETED -980__.c:MIGRATED -980__c:MERGED + """ + + __ignore_keys__ = IGNORE_SYSTEM_KEYS | { + "030__a", # coden designation + "260__b", # Always CERN + "300__b", # Resolution of the video + "340__a", # Physical medium + "518__h", # Start time of meeting/conference event + "518__g", # Meeting/conference identification + "520__9", # Source of the additional description (e.g. arxiv) + "542__3", # Part of the license + "595__i", # INSPEC number + "773__a", # Duplicate DOI + "901__u", # Affiliation at Conversion? + "913__y", # citation + "913__v", # citation + "913__t", # citation + "913__a", # citation + "913__c", # citation + "964__a", # number of physical copies + "970__b", # spreadsheet + } + + _default_fields = { + "custom_fields": {}, + } + + +at_model = ATModel( + bases=(rdm_base_publication_model,), + entry_point_group="cds_migrator_kit.migrator.rules.at", +) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py new file mode 100644 index 00000000..b461d318 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. +# +from dojson.errors import IgnoreKey + +from cds_migrator_kit.transform.xml_processing.quality.decorators import ( + for_each_value, + require, +) +from cds_migrator_kit.transform.xml_processing.quality.parsers import StringValue + +from ...models.at import at_model as model + + +@model.over("contributors", "^541__") +@for_each_value +def contact_person(self, key, value): + contact_person = value.get("a", None) + if contact_person is None: + raise IgnoreKey("contributors") + + contact_person = StringValue(contact_person).parse() + + # Sometimes the contact person is stored as ", " where the role is some position within the project. + parts = contact_person.split(",") + if len(parts) == 2: + name, role = parts + contact_person = name.strip() + + contributor = { + "person_or_org": { + "type": "personal", + "name": contact_person, + "family_name": contact_person, + }, + "role": {"id": "contactperson"}, + } + return contributor diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index e40379d8..4d2e6c4c 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -1053,10 +1053,25 @@ def imprint_info(self, key, value): if place: imprint["place"] = place.rstrip(".") self["custom_fields"]["imprint:imprint"] = imprint + if publication_date_str: try: - publication_date = normalize(publication_date_str) + # Sometimes the imprint publication date ends with a "?" to mean that it's uncertain + # when the exact date was. + if "?" in publication_date_str: + publication_date_str = ( + publication_date_str.replace("?", "").rstrip("-").strip() + ) + self.setdefault("dates", []).append( + { + "description": "The publication date is indeterminate.", + "date": normalize(publication_date_str), + "type": {"id": "created"}, + } + ) + # TODO: should we still set as the main publication date if it's uncertain? + publication_date = normalize(publication_date_str) self["publication_date"] = publication_date except (ParserError, TypeError) as e: raise UnexpectedValue( diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index cb1d4805..d2be83c7 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -8,7 +8,11 @@ from idutils.normalizers import normalize_isbn, normalize_issn from isbnlib import NotValidISBNError -from cds_migrator_kit.errors import ManualImportRequired, RecordFlaggedCuration, UnexpectedValue +from cds_migrator_kit.errors import ( + ManualImportRequired, + RecordFlaggedCuration, + UnexpectedValue, +) from cds_migrator_kit.transform.xml_processing.quality.decorators import ( filter_list_values, for_each_value, @@ -21,8 +25,9 @@ udc_pattern, ) from ...models.base_publication_record import rdm_base_publication_model as model -from .base import normalize from ..quality.reviewers import find_reviewer +from .base import licenses as _base_licenses +from .base import normalize @model.over("isbns", "^020__", override_tag=True) @@ -104,7 +109,7 @@ def corpo_author(self, key, value): return author raise IgnoreKey("creators") -1 + @model.over("imprint_info", "(^250__)") @for_each_value @require(["a"]) @@ -277,7 +282,7 @@ def journal(self, key, value): new_meeting = {} identifiers = [] if conference_url: - identifiers.append({"scheme": "URL", "identifier": conference_url}) + identifiers.append({"scheme": "URL", "identifier": conference_url}) if conference_cnum: identifiers.append({"scheme": "inspire", "identifier": conference_cnum}) new_meeting["identifiers"] = identifiers @@ -300,8 +305,13 @@ def journal(self, key, value): if not is_journal_year and "y" in value: if not pub_date: self["publication_date"] = year + # Check if the pub date and year match as raw strings; if not, check if the year part of the pub date matches the conference year elif pub_date != year: - raise UnexpectedValue("Publication date mismatch", field=key, value=value) + parsed_pub_date = parse(pub_date, default=datetime(1, 1, 1), dayfirst=True) + if parsed_pub_date.year != int(year): + raise UnexpectedValue( + "Publication date mismatch", field=key, value=value + ) # Only populate journal fields from a journal 773 (has p/n/v). # A 773 with only 'c'+'w' is a conference proceedings reference and must @@ -606,7 +616,6 @@ def resource_type(self, key, value): # handles occurrences where document type should have been an experiment add_experiment_for = {"lhcbcerntalk": "LHCb", "lhcb_misc": "LHCb"} - committees = { "scicommpubldrdc": "DRDC", "scicommpubleec": "EEC", @@ -631,9 +640,7 @@ def resource_type(self, key, value): value_a = value_a.lower() value_b = value_b.lower() - if (value_a in committees.keys()) or ( - value_b in committees - ): + if (value_a in committees.keys()) or (value_b in committees): custom_fields = self.get("custom_fields", {}) comm_cf = custom_fields.get("cern:committees", []) if value_a: @@ -643,12 +650,9 @@ def resource_type(self, key, value): self["custom_fields"]["cern:committees"] = comm_cf raise IgnoreKey("resource_type") - if (value_a.lower() in ignore_res_types) or ( - value_b in ignore_res_types - ): + if (value_a.lower() in ignore_res_types) or (value_b in ignore_res_types): raise IgnoreKey("resource_type") - if value_a in add_experiment_for.keys(): custom_field = self.get("custom_fields", {}) experiments = custom_field.get("cern:experiments", []) diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 06be0344..5add2e69 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -340,6 +340,17 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "" + at: + data_dir: cds_migrator_kit/rdm/data/at + restricted: "False" + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/at/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/at/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "2ebd631f-66c3-4891-b2dd-275ec3656b7b" faser-drafts: data_dir: cds_migrator_kit/rdm/data/faser-drafts restricted: "True" @@ -668,13 +679,13 @@ comments: faser-drafts: dir_path: /migration/faser-drafts/comments/ reviewers: - - {group: faser-all} + - { group: faser-all } faser: dir_path: /migration/faser/comments/ reviewers: - - {group: faser-all} + - { group: faser-all } faser-ep: dir_path: /migration/faser-ep/comments/ reviewers: - - {group: faser-all} + - { group: faser-all } diff --git a/setup.cfg b/setup.cfg index 8d5e37ec..82972026 100644 --- a/setup.cfg +++ b/setup.cfg @@ -92,6 +92,7 @@ cds_migrator_kit.migrator.models = sy = cds_migrator_kit.rdm.records.transform.models.sy:sy_model te = cds_migrator_kit.rdm.records.transform.models.te:te_model en = cds_migrator_kit.rdm.records.transform.models.en:en_model + at = cds_migrator_kit.rdm.records.transform.models.at:at_model annual_rep = cds_migrator_kit.rdm.records.transform.models.annual_report:annual_rep_model bulletin_drafts = cds_migrator_kit.rdm.records.transform.models.bulletin_drafts:bulletin_drafts_model staff_association = cds_migrator_kit.rdm.records.transform.models.staff_association:staff_association_model @@ -193,6 +194,11 @@ cds_migrator_kit.migrator.rules.fap = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base fap = cds_migrator_kit.rdm.records.transform.xml_processing.rules.fap +cds_migrator_kit.migrator.rules.at = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research + at = cds_migrator_kit.rdm.records.transform.xml_processing.rules.at cds_migrator_kit.migrator.rules.bulletin_drafts = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base