add(models): add AT model

cds_migrator_kit/rdm/records/transform/models/_config.py

@@ -1,10 +1,12 @@
 """Fields which we can confidently ignore in each model."""
+
 IGNORE_SYSTEM_KEYS = {
     "0248_a",
     "0248_p",
     "0248_q",
     "852__c",  # holdings will be taken separately
     "852__h",
+    "035__z",  # arxiv identifier (e.g. https://cds.cern.ch/record/1054291/export/hm?no_redirect_migrated)
     "037__c",  # arxiv subject
     "100__m",  # email of contributor
     "300__a",  # number of pages
@@ -13,6 +15,8 @@
     "8564_s",  # bibdoc id
     "8564_x",  # icon thumbnails sizes
     "8564_y",  # file description - done by files dump
+    "8564_8",  # File information (done by file dump)
+    "8564_q",  # File links File information (done by file dump)
     "916__y",  # year, redundant value
     "937__c",  # last modified by
     "937__s",  # last modification date

cds_migrator_kit/rdm/records/transform/models/at.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS
+from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
+    rdm_base_publication_model,
+)
+from cds_migrator_kit.transform.overdo import CdsOverdo
+
+
+class ATModel(CdsOverdo):
+    """Translation model for AT records."""
+
+    __query__ = """
+        (980__:ARTICLE OR 980__:PREPRINT)
+        AND
+        (710__.5:NPA OR 710__.5:AT)
+        -710__.5:SI -710__.5:SC -710__.5:SL -710__.5:PS -710__.5:MPS -710__.5:ISR -710__.5:MSC -710__.5:AC -710__.5:SPS
+        -710__.5:LEP -710__.5:AB -710__.5:AR -710__.5:TS -710__.5:ST -710__.5:MT -710__.5:EST -710__.5:SB -710__.5:LHC
+        -980__:DELETED -980__.c:MIGRATED -980__c:MERGED
+    """
+
+    __ignore_keys__ = IGNORE_SYSTEM_KEYS | {
+        "030__a",  # coden designation
+        "260__b",  # Always CERN
+        "300__b",  # Resolution of the video
+        "340__a",  # Physical medium
+        "518__h",  # Start time of meeting/conference event
+        "518__g",  # Meeting/conference identification
+        "520__9",  # Source of the additional description (e.g. arxiv)
+        "542__3",  # Part of the license
+        "595__i",  # INSPEC number
+        "773__a",  # Duplicate DOI
+        "901__u",  # Affiliation at Conversion?
+        "913__y",  # citation
+        "913__v",  # citation
+        "913__t",  # citation
+        "913__a",  # citation
+        "913__c",  # citation
+        "964__a",  # number of physical copies
+        "970__b",  # spreadsheet
+    }
+
+    _default_fields = {
+        "custom_fields": {},
+    }
+
+
+at_model = ATModel(
+    bases=(rdm_base_publication_model,),
+    entry_point_group="cds_migrator_kit.migrator.rules.at",
+)

cds_migrator_kit/rdm/records/transform/xml_processing/rules/at.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+#
+from dojson.errors import IgnoreKey
+
+from cds_migrator_kit.transform.xml_processing.quality.decorators import (
+    for_each_value,
+    require,
+)
+from cds_migrator_kit.transform.xml_processing.quality.parsers import StringValue
+
+from ...models.at import at_model as model
+
+
+@model.over("contributors", "^541__")
+@for_each_value
+def contact_person(self, key, value):
+    contact_person = value.get("a", None)
+    if contact_person is None:
+        raise IgnoreKey("contributors")
+
+    contact_person = StringValue(contact_person).parse()
+
+    # Sometimes the contact person is stored as "<Name>, <Role>" where the role is some position within the project.
+    parts = contact_person.split(",")
+    if len(parts) == 2:
+        name, role = parts
+        contact_person = name.strip()
+
+    contributor = {
+        "person_or_org": {
+            "type": "personal",
+            "name": contact_person,
+            "family_name": contact_person,
+        },
+        "role": {"id": "contactperson"},
+    }
+    return contributor

cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py

@@ -1053,10 +1053,25 @@ def imprint_info(self, key, value):
     if place:
         imprint["place"] = place.rstrip(".")
     self["custom_fields"]["imprint:imprint"] = imprint
+
     if publication_date_str:
         try:
-            publication_date = normalize(publication_date_str)
+            # Sometimes the imprint publication date ends with a "?" to mean that it's uncertain
+            # when the exact date was.
+            if "?" in publication_date_str:
+                publication_date_str = (
+                    publication_date_str.replace("?", "").rstrip("-").strip()
+                )
+                self.setdefault("dates", []).append(
+                    {
+                        "description": "The publication date is indeterminate.",
+                        "date": normalize(publication_date_str),
+                        "type": {"id": "created"},
+                    }
+                )
 
+            # TODO: should we still set as the main publication date if it's uncertain?
+            publication_date = normalize(publication_date_str)
             self["publication_date"] = publication_date
         except (ParserError, TypeError) as e:
             raise UnexpectedValue(

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py

@@ -8,7 +8,11 @@
 from idutils.normalizers import normalize_isbn, normalize_issn
 from isbnlib import NotValidISBNError
 
-from cds_migrator_kit.errors import ManualImportRequired, RecordFlaggedCuration, UnexpectedValue
+from cds_migrator_kit.errors import (
+    ManualImportRequired,
+    RecordFlaggedCuration,
+    UnexpectedValue,
+)
 from cds_migrator_kit.transform.xml_processing.quality.decorators import (
     filter_list_values,
     for_each_value,
@@ -21,8 +25,9 @@
     udc_pattern,
 )
 from ...models.base_publication_record import rdm_base_publication_model as model
-from .base import normalize
 from ..quality.reviewers import find_reviewer
+from .base import licenses as _base_licenses
+from .base import normalize
 
 
 @model.over("isbns", "^020__", override_tag=True)
@@ -104,7 +109,7 @@ def corpo_author(self, key, value):
         return author
     raise IgnoreKey("creators")
 
-1
+
 @model.over("imprint_info", "(^250__)")
 @for_each_value
 @require(["a"])
@@ -277,7 +282,7 @@ def journal(self, key, value):
         new_meeting = {}
         identifiers = []
         if conference_url:
-            identifiers.append({"scheme": "URL", "identifier":  conference_url})
+            identifiers.append({"scheme": "URL", "identifier": conference_url})
         if conference_cnum:
             identifiers.append({"scheme": "inspire", "identifier": conference_cnum})
             new_meeting["identifiers"] = identifiers
@@ -300,8 +305,13 @@ def journal(self, key, value):
     if not is_journal_year and "y" in value:
         if not pub_date:
             self["publication_date"] = year
+        # Check if the pub date and year match as raw strings; if not, check if the year part of the pub date matches the conference year
         elif pub_date != year:
-            raise UnexpectedValue("Publication date mismatch", field=key, value=value)
+            parsed_pub_date = parse(pub_date, default=datetime(1, 1, 1), dayfirst=True)
+            if parsed_pub_date.year != int(year):
+                raise UnexpectedValue(
+                    "Publication date mismatch", field=key, value=value
+                )
 
     # Only populate journal fields from a journal 773 (has p/n/v).
     # A 773 with only 'c'+'w' is a conference proceedings reference and must
@@ -606,7 +616,6 @@ def resource_type(self, key, value):
     # handles occurrences where document type should have been an experiment
     add_experiment_for = {"lhcbcerntalk": "LHCb", "lhcb_misc": "LHCb"}
 
-
     committees = {
         "scicommpubldrdc": "DRDC",
         "scicommpubleec": "EEC",
@@ -631,9 +640,7 @@ def resource_type(self, key, value):
     value_a = value_a.lower()
     value_b = value_b.lower()
 
-    if (value_a in committees.keys()) or (
-        value_b in committees
-    ):
+    if (value_a in committees.keys()) or (value_b in committees):
         custom_fields = self.get("custom_fields", {})
         comm_cf = custom_fields.get("cern:committees", [])
         if value_a:
@@ -643,12 +650,9 @@ def resource_type(self, key, value):
         self["custom_fields"]["cern:committees"] = comm_cf
         raise IgnoreKey("resource_type")
 
-    if (value_a.lower() in ignore_res_types) or (
-        value_b in ignore_res_types
-    ):
+    if (value_a.lower() in ignore_res_types) or (value_b in ignore_res_types):
         raise IgnoreKey("resource_type")
 
-
     if value_a in add_experiment_for.keys():
         custom_field = self.get("custom_fields", {})
         experiments = custom_field.get("cern:experiments", [])

cds_migrator_kit/rdm/streams.yaml

@@ -340,6 +340,17 @@ records:
       missing_users: cds_migrator_kit/rdm/data/users
       communities_ids:
         - ""
+  at:
+    data_dir: cds_migrator_kit/rdm/data/at
+    restricted: "False"
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/at/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/at/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - "2ebd631f-66c3-4891-b2dd-275ec3656b7b"
   faser-drafts:
     data_dir: cds_migrator_kit/rdm/data/faser-drafts
     restricted: "True"
@@ -668,13 +679,13 @@ comments:
   faser-drafts:
     dir_path: /migration/faser-drafts/comments/
     reviewers:
-      - {group: faser-all}
+      - { group: faser-all }
   faser:
     dir_path: /migration/faser/comments/
     reviewers:
-      - {group: faser-all}
+      - { group: faser-all }
 
   faser-ep:
     dir_path: /migration/faser-ep/comments/
     reviewers:
-      - {group: faser-all}
+      - { group: faser-all }

setup.cfg

@@ -92,6 +92,7 @@ cds_migrator_kit.migrator.models =
     sy = cds_migrator_kit.rdm.records.transform.models.sy:sy_model
     te = cds_migrator_kit.rdm.records.transform.models.te:te_model
     en = cds_migrator_kit.rdm.records.transform.models.en:en_model
+    at = cds_migrator_kit.rdm.records.transform.models.at:at_model
     annual_rep = cds_migrator_kit.rdm.records.transform.models.annual_report:annual_rep_model
     bulletin_drafts = cds_migrator_kit.rdm.records.transform.models.bulletin_drafts:bulletin_drafts_model
     staff_association = cds_migrator_kit.rdm.records.transform.models.staff_association:staff_association_model
@@ -193,6 +194,11 @@ cds_migrator_kit.migrator.rules.fap =
     base = cds_migrator_kit.transform.xml_processing.rules.base
     base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base
     fap = cds_migrator_kit.rdm.records.transform.xml_processing.rules.fap
+cds_migrator_kit.migrator.rules.at =
+    base = cds_migrator_kit.transform.xml_processing.rules.base
+    base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base
+    publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research
+    at = cds_migrator_kit.rdm.records.transform.xml_processing.rules.at
 cds_migrator_kit.migrator.rules.bulletin_drafts =
     base = cds_migrator_kit.transform.xml_processing.rules.base
     base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base