From d1ad4f0e2bf2911ea4d94e39c604fdf8215791c4 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Tue, 9 Jan 2024 13:23:52 -0500
Subject: [PATCH 001/137] #10218 OdumInstitute is now uncch-rdmc

---
 .github/workflows/guides_build_sphinx.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml
index 992f30f2872..86b59b11d35 100644
--- a/.github/workflows/guides_build_sphinx.yml
+++ b/.github/workflows/guides_build_sphinx.yml
@@ -11,6 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - uses: OdumInstitute/sphinx-action@master
+    - uses: uncch-rdmc/sphinx-action@master
       with:
         docs-folder: "doc/sphinx-guides/"

From fd9f051540f0aab5db340a01d74bd38d9a0dee27 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Tue, 9 Jan 2024 13:25:10 -0500
Subject: [PATCH 002/137] #10218 OdumInstitute is now uncch-rdmc

---
 doc/sphinx-guides/source/developers/version-control.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst
index 12f3d5b81fd..ce10a159b42 100644
--- a/doc/sphinx-guides/source/developers/version-control.rst
+++ b/doc/sphinx-guides/source/developers/version-control.rst
@@ -274,16 +274,16 @@ By default, when a pull request is made from a fork, "Allow edits from maintaine
 
 This is a nice feature of GitHub because it means that the core dev team for the Dataverse Project can make small (or even large) changes to a pull request from a contributor to help the pull request along on its way to QA and being merged.
 
-GitHub documents how to make changes to a fork at https://help.github.com/articles/committing-changes-to-a-pull-request-branch-created-from-a-fork/ but as of this writing the steps involve making a new clone of the repo. This works but you might find it more convenient to add a "remote" to your existing clone. The example below uses the fork at https://github.com/OdumInstitute/dataverse and the branch ``4709-postgresql_96`` but the technique can be applied to any fork and branch:
+GitHub documents how to make changes to a fork at https://help.github.com/articles/committing-changes-to-a-pull-request-branch-created-from-a-fork/ but as of this writing the steps involve making a new clone of the repo. This works but you might find it more convenient to add a "remote" to your existing clone. The example below uses the fork at https://github.com/uncch-rdmc/dataverse and the branch ``4709-postgresql_96`` but the technique can be applied to any fork and branch:
 
 .. code-block:: bash
 
-        git remote add OdumInstitute git@github.com:OdumInstitute/dataverse.git
-        git fetch OdumInstitute
+        git remote add uncch-rdmc git@github.com:uncch-rdmc/dataverse.git
+        git fetch uncch-rdmc
         git checkout 4709-postgresql_96
         vim path/to/file.txt
         git commit
-        git push OdumInstitute 4709-postgresql_96
+        git push uncch-rdmc 4709-postgresql_96
 
 ----
 

From c4b82dfb830ba014b79552830af86a9940648456 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Feb 2024 15:44:21 +0100
Subject: [PATCH 003/137] fix(deps): resolve conflicting version of MIME4J
 #9077

- Apache Abdera Parser, Apache Tika and RESTeasy (Testing) use MIME4J
- Tika and RESTeasy use newer APIs only present since v0.8+
- Abdera is an abandoned project, uses v0.7.2 and is hopefully
  compatible with newer releases
- v0.8.4 given by Apache Tika relies on vulnerable Apache Commons IO
  2.6, we want 2.11 per dependency management. Upgrading to v0.8.7 as
  earliest version with 2.11 dependency
---
 pom.xml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/pom.xml b/pom.xml
index f45e8fd9033..07d73336095 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,6 +51,17 @@
                 <artifactId>abdera-i18n</artifactId>
                 <version>1.1.3</version>
             </dependency>
+            <!-- Dependency for Apache Abdera and Apache Tika. Tika needs newer version. -->
+            <dependency>
+                <groupId>org.apache.james</groupId>
+                <artifactId>apache-mime4j-core</artifactId>
+                <version>0.8.7</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.james</groupId>
+                <artifactId>apache-mime4j-dom</artifactId>
+                <version>0.8.7</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
     <!-- Declare any DIRECT dependencies here.

From 477eb068ab411699256008a2ed4b60a67f1d4cb8 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 7 Feb 2024 07:47:18 +0100
Subject: [PATCH 004/137] chore(deps): update Apache Tika to v2.9.1

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 07d73336095..b04d8a9e10a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
         <flyway.version>9.22.1</flyway.version>
         <jhove.version>1.20.1</jhove.version>
         <poi.version>5.2.1</poi.version>
-        <tika.version>2.4.1</tika.version>
+        <tika.version>2.9.1</tika.version>
         <netcdf.version>5.5.3</netcdf.version>
     </properties>
     

From f3f86a861de29f9e05aa08272fd3f250de831e87 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 28 Jun 2024 18:01:08 +0000
Subject: [PATCH 005/137] Bump org.xmlunit:xmlunit-core from 2.9.1 to 2.10.0

Bumps [org.xmlunit:xmlunit-core](https://github.com/xmlunit/xmlunit) from 2.9.1 to 2.10.0.
- [Release notes](https://github.com/xmlunit/xmlunit/releases)
- [Changelog](https://github.com/xmlunit/xmlunit/blob/main/RELEASE_NOTES.md)
- [Commits](https://github.com/xmlunit/xmlunit/compare/v2.9.1...v2.10.0)

---
updated-dependencies:
- dependency-name: org.xmlunit:xmlunit-core
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 76a8f61444f..fa99adca8b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -553,7 +553,7 @@
         <dependency>
             <groupId>org.xmlunit</groupId>
             <artifactId>xmlunit-core</artifactId>
-            <version>2.9.1</version>
+            <version>2.10.0</version>
         </dependency>
         <dependency>
           <groupId>com.google.cloud</groupId>

From fd82437b882311491be252382b8dce0786a0c824 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Thu, 27 Jun 2024 18:15:58 -0400
Subject: [PATCH 006/137] refactor/fix avoidJoin logic

---
 .../dataverse/search/SearchServiceBean.java   | 245 ++++--------------
 .../iq/dataverse/search/SearchUtil.java       |   1 +
 2 files changed, 53 insertions(+), 193 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 1d25dbcdaba..ad02f8c49ac 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -42,6 +42,8 @@
 import jakarta.ejb.TransactionRolledbackLocalException;
 import jakarta.inject.Named;
 import jakarta.persistence.NoResultException;
+
+import org.apache.commons.lang3.StringUtils;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrQuery.SortClause;
 import org.apache.solr.client.solrj.SolrServerException;
@@ -187,6 +189,7 @@ public SolrQueryResponse search(
 
         SolrQuery solrQuery = new SolrQuery();
         query = SearchUtil.sanitizeQuery(query);
+
         solrQuery.setQuery(query);
         if (sortField != null) {
             // is it ok not to specify any sort? - there are cases where we 
@@ -327,24 +330,13 @@ public SolrQueryResponse search(
             }
         }
 
-        //I'm not sure if just adding null here is good for hte permissions system... i think it needs something
-        if(dataverses != null) {
-            for(Dataverse dataverse : dataverses) {
-                // -----------------------------------
-                // PERMISSION FILTER QUERY
-                // -----------------------------------
-                String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, dataverse, onlyDatatRelatedToMe, addFacets);
-                if (permissionFilterQuery != null) {
-                    solrQuery.addFilterQuery(permissionFilterQuery);
-                }
-            }
-        } else {
-            String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, null, onlyDatatRelatedToMe, addFacets);
-            if (permissionFilterQuery != null) {
-                solrQuery.addFilterQuery(permissionFilterQuery);
-            }
+        // -----------------------------------
+        // PERMISSION FILTER QUERY
+        // -----------------------------------
+        String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, onlyDatatRelatedToMe, addFacets);
+        if (permissionFilterQuery != null) {
+            solrQuery.addFilterQuery(permissionFilterQuery);
         }
-
         
         /**
          * @todo: do sanity checking... throw error if negative
@@ -976,7 +968,7 @@ public String getCapitalizedName(String name) {
      *
      * @return
      */
-    private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQuery solrQuery, Dataverse dataverse, boolean onlyDatatRelatedToMe, boolean addFacets) {
+    private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQuery solrQuery, boolean onlyDatatRelatedToMe, boolean addFacets) {
 
         User user = dataverseRequest.getUser();
         if (user == null) {
@@ -985,38 +977,23 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
         if (solrQuery == null) {
             throw new NullPointerException("solrQuery cannot be null");
         }
-        /**
-         * @todo For people who are not logged in, should we show stuff indexed
-         * with "AllUsers" group or not? If so, uncomment the allUsersString
-         * stuff below.
-         */
-//        String allUsersString = IndexServiceBean.getGroupPrefix() + AllUsers.get().getAlias();
-//        String publicOnly = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":(" + IndexServiceBean.getPublicGroupString() + " OR " + allUsersString + ")";
-        String publicOnly = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":(" + IndexServiceBean.getPublicGroupString() + ")";
-//        String publicOnly = "{!join from=" + SearchFields.GROUPS + " to=" + SearchFields.PERMS + "}id:" + IndexServiceBean.getPublicGroupString();
-        // initialize to public only to be safe
+        
         String dangerZoneNoSolrJoin = null;
 
         if (user instanceof PrivateUrlUser) {
             user = GuestUser.get();
         }
 
-        AuthenticatedUser au = null; 
+        ArrayList<String> groupList = new ArrayList<String>();
+        AuthenticatedUser au = null;
         Set<Group> groups;
-        
-        if (user instanceof GuestUser) {
-            // Yes, GuestUser may be part of one or more groups; such as IP Groups.
-            groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
-        } else {
-            if (!(user instanceof AuthenticatedUser)) {
-                logger.severe("Should never reach here. A User must be an AuthenticatedUser or a Guest");
-                throw new IllegalStateException("A User must be an AuthenticatedUser or a Guest");
-            }
+
+        if (user instanceof AuthenticatedUser) {
 
             au = (AuthenticatedUser) user;
-            
+
             // ----------------------------------------------------
-            // (3) Is this a Super User?
+            // Is this a Super User?
             // If so, they can see everything
             // ----------------------------------------------------
             if (au.isSuperuser()) {
@@ -1026,11 +1003,11 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
 
                 return dangerZoneNoSolrJoin;
             }
-           
+
             // ----------------------------------------------------
-            // (4) User is logged in AND onlyDatatRelatedToMe == true
+            // User is logged in AND onlyDatatRelatedToMe == true
             // Yes, give back everything -> the settings will be in
-            //          the filterqueries given to search
+            // the filterqueries given to search
             // ----------------------------------------------------
             if (onlyDatatRelatedToMe == true) {
                 if (systemConfig.myDataDoesNotUsePermissionDocs()) {
@@ -1041,169 +1018,51 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
                     logger.fine("new post-4.2 behavior: MyData is using Solr permission docs");
                 }
             }
-            
             // ----------------------------------------------------
-            // (5) Work with Authenticated User who is not a Superuser
+            // Work with Authenticated User who is not a Superuser
             // ----------------------------------------------------
-
-            groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
+            groupList.add(IndexServiceBean.getGroupPerUserPrefix() + au.getId());
         }
         
-        if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) {
-            /**
-             * Instead of doing a super expensive join, we will rely on the 
-             * new boolean field PublicObject:true for public objects. This field 
-             * is indexed on the content document itself, rather than a permission 
-             * document. An additional join will be added only for any extra, 
-             * more restricted groups that the user may be part of.
-             * **Note the experimental nature of this optimization**. 
-             */
-            StringBuilder sb = new StringBuilder();
-            StringBuilder sbgroups = new StringBuilder();
-            
-            // All users, guests and authenticated, should see all the 
-            // documents marked as publicObject_b:true, at least:
-            sb.append(SearchFields.PUBLIC_OBJECT + ":" + true);
+        // In addition to the user referenced directly, we will also
+        // add joins on all the non-public groups that may exist for the
+        // user:
 
-            // One or more groups *may* also be available for this user. Once again,
-            // do note that Guest users may be part of some groups, such as 
-            // IP groups. 
-            
-            int groupCounter = 0;
-
-            // An AuthenticatedUser should also be able to see all the content 
-            // on which they have direct permissions:              
-            if (au != null) {
-                groupCounter++; 
-                sbgroups.append(IndexServiceBean.getGroupPerUserPrefix() + au.getId());
-            }
- 
-            // In addition to the user referenced directly, we will also 
-            // add joins on all the non-public groups that may exist for the
-            // user:
-            for (Group group : groups) {
-                String groupAlias = group.getAlias();
-                if (groupAlias != null && !groupAlias.isEmpty() && !groupAlias.startsWith("builtIn")) {
-                    groupCounter++;
-                    if (groupCounter > 1) {
-                        sbgroups.append(" OR ");
-                    }
-                    sbgroups.append(IndexServiceBean.getGroupPrefix() + groupAlias);
-                }
-            }
-            
-            if (groupCounter > 1) {
-                // If there is more than one group, the parentheses must be added:
-                sbgroups.insert(0, "(");
-                sbgroups.append(")");
-            }
-            
-            if (groupCounter > 0) {
-                // If there are any groups for this user, an extra join must be
-                // added to the query, and the extra sub-query must be added to 
-                // the combined Solr query:
-                sb.append(" OR {!join from=" + SearchFields.DEFINITION_POINT + " to=id v=$q1}");
-                // Add the subquery to the combined Solr query: 
-                solrQuery.setParam("q1", SearchFields.DISCOVERABLE_BY + ":" + sbgroups.toString());
-                logger.info("The sub-query q1 set to " + SearchFields.DISCOVERABLE_BY + ":" + sbgroups.toString());
-            }
-            
-            String ret = sb.toString();
-            logger.fine("Returning experimental query: " + ret);
-            return ret;
-        }
-        
-        // END OF EXPERIMENTAL OPTIMIZATION 
+        boolean avoidJoin = FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled();
         
-        // Old, un-optimized way of handling permissions.
-        // Largely left intact, minus the lookups that have already been performed
-        // above.
-        
-        // ----------------------------------------------------
-        // (1) Is this a GuestUser?
-        // ----------------------------------------------------
-        if (user instanceof GuestUser) {
-            
-            StringBuilder sb = new StringBuilder();
-            
-            String groupsFromProviders = "";
-            for (Group group : groups) {
-                logger.fine("found group " + group.getIdentifier() + " with alias " + group.getAlias());
-                String groupAlias = group.getAlias();
-                if (groupAlias != null && !groupAlias.isEmpty()) {
-                    sb.append(" OR ");
-                    // i.e. group_builtIn/all-users, ip/ipGroup3
-                    sb.append(IndexServiceBean.getGroupPrefix()).append(groupAlias);
-                }
-            }
-            groupsFromProviders = sb.toString();
-            logger.fine("groupsFromProviders:" + groupsFromProviders);
-            String guestWithGroups = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":(" + IndexServiceBean.getPublicGroupString() + groupsFromProviders + ")";
-            logger.fine(guestWithGroups);
-            return guestWithGroups;
-        }
+        // Authenticated users and GuestUser may be part of one or more groups; such
+        // as IP Groups.
+        groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
 
-        // ----------------------------------------------------
-        // (5) Work with Authenticated User who is not a Superuser
-        // ----------------------------------------------------  
-        // It was already confirmed, that if the user is not GuestUser, we 
-        // have an AuthenticatedUser au which is not null. 
-        /**
-         * @todo all this code needs cleanup and clarification.
-         */
-        /**
-         * Every AuthenticatedUser is part of a "User Private Group" (UGP), a
-         * concept we borrow from RHEL:
-         * https://access.redhat.com/site/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/ch-Managing_Users_and_Groups.html#s2-users-groups-private-groups
-         */
-        /**
-         * @todo rename this from publicPlusUserPrivateGroup. Confusing
-         */
-        // safe default: public only
-        String publicPlusUserPrivateGroup = publicOnly;
-//                    + (onlyDatatRelatedToMe ? "" : (publicOnly + " OR "))
-//                    + "{!join from=" + SearchFields.GROUPS + " to=" + SearchFields.PERMS + "}id:" + IndexServiceBean.getGroupPerUserPrefix() + au.getId() + ")";
-
-//            /**
-//             * @todo add onlyDatatRelatedToMe option into the experimental JOIN
-//             * before enabling it.
-//             */
-        /**
-         * From a search perspective, we don't care about if the group was
-         * created within one dataverse or another. We just want a list of *all*
-         * the groups the user is part of. We are greedy. We want all BuiltIn
-         * Groups, Shibboleth Groups, IP Groups, "system" groups, everything.
-         *
-         * A JOIN on "permission documents" will determine if the user can find
-         * a given "content document" (dataset version, etc) in Solr.
-         */
-        String groupsFromProviders = "";
-        StringBuilder sb = new StringBuilder();
         for (Group group : groups) {
-            logger.fine("found group " + group.getIdentifier() + " with alias " + group.getAlias());
             String groupAlias = group.getAlias();
-            if (groupAlias != null && !groupAlias.isEmpty()) {
-                sb.append(" OR ");
-                // i.e. group_builtIn/all-users, group_builtIn/authenticated-users, group_1-explictGroup1, group_shib/2
-                sb.append(IndexServiceBean.getGroupPrefix() + groupAlias);
+            if (groupAlias != null && !groupAlias.isEmpty() && (!avoidJoin || !groupAlias.startsWith("builtIn"))) {
+                groupList.add(IndexServiceBean.getGroupPrefix() + groupAlias);
             }
         }
-        groupsFromProviders = sb.toString();
 
-        logger.fine(groupsFromProviders);
-        if (true) {
-            /**
-             * @todo get rid of "experimental" in name
-             */
-            String experimentalJoin = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":(" + IndexServiceBean.getPublicGroupString() + " OR " + IndexServiceBean.getGroupPerUserPrefix() + au.getId() + groupsFromProviders + ")";
-            publicPlusUserPrivateGroup = experimentalJoin;
+        if (!avoidJoin) {
+            // Add the public group
+            groupList.add(0, IndexServiceBean.getPublicGroupString());
+        } 
+        
+        String groupString = null;
+        //If we have additional groups, format them correctly into a search string, with parens if there is more than one
+        if (groupList.size() > 1) {
+            groupString = "(" + StringUtils.join(groupList, " OR ") + ")";
+        } else if (groupList.size() == 1) {
+            groupString = groupList.get(0);
         }
-
-        //permissionFilterQuery = publicPlusUserPrivateGroup;
-        logger.fine(publicPlusUserPrivateGroup);
-
-        return publicPlusUserPrivateGroup;
-
+        logger.fine("Groups: " + groupString);
+        String query = avoidJoin ? SearchFields.PUBLIC_OBJECT + ":" + true : "";
+        if (groupString != null) {
+            if (!query.isEmpty()) {
+                query = "(" + query + " OR " + "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + groupString + ")";
+            } else {
+                query = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + groupString;
+            }
+        }
+        return query;
     }
 
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java
index adcc5825766..935a003e0f7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java
@@ -235,4 +235,5 @@ public static String getGeoRadius(String userSuppliedGeoRadius) throws NumberFor
         return userSuppliedGeoRadius;
     }
 
+
 }

From 0331585755107f93c0bfc7abb0ff6117d2a1c2b1 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 3 Jul 2024 12:51:49 -0400
Subject: [PATCH 007/137] bug fix - default to */All groups for superuser

---
 .../dataverse/search/SearchServiceBean.java   | 30 +++++++++++--------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index ad02f8c49ac..6f07da8a96f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -61,6 +61,8 @@ public class SearchServiceBean {
 
     private static final Logger logger = Logger.getLogger(SearchServiceBean.class.getCanonicalName());
 
+    private static final String ALL_GROUPS = "*";
+
     /**
      * We're trying to make the SearchServiceBean lean, mean, and fast, with as
      * few injections of EJBs as possible.
@@ -978,8 +980,6 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
             throw new NullPointerException("solrQuery cannot be null");
         }
         
-        String dangerZoneNoSolrJoin = null;
-
         if (user instanceof PrivateUrlUser) {
             user = GuestUser.get();
         }
@@ -988,8 +988,9 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
         AuthenticatedUser au = null;
         Set<Group> groups;
 
-        if (user instanceof AuthenticatedUser) {
+        boolean avoidJoin = FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled();
 
+        if (user instanceof AuthenticatedUser) {
             au = (AuthenticatedUser) user;
 
             // ----------------------------------------------------
@@ -1001,7 +1002,7 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
                 // to see everything in Solr with no regard to permissions. But it's
                 // been this way since Dataverse 4.0. So relax. :)
 
-                return dangerZoneNoSolrJoin;
+                return buildPermissionFilterQuery(avoidJoin, ALL_GROUPS);
             }
 
             // ----------------------------------------------------
@@ -1012,7 +1013,7 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
             if (onlyDatatRelatedToMe == true) {
                 if (systemConfig.myDataDoesNotUsePermissionDocs()) {
                     logger.fine("old 4.2 behavior: MyData is not using Solr permission docs");
-                    return dangerZoneNoSolrJoin;
+                    return buildPermissionFilterQuery(avoidJoin, ALL_GROUPS);
                 } else {
                     // fall-through
                     logger.fine("new post-4.2 behavior: MyData is using Solr permission docs");
@@ -1028,8 +1029,6 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
         // add joins on all the non-public groups that may exist for the
         // user:
 
-        boolean avoidJoin = FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled();
-        
         // Authenticated users and GuestUser may be part of one or more groups; such
         // as IP Groups.
         groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
@@ -1054,16 +1053,23 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
             groupString = groupList.get(0);
         }
         logger.fine("Groups: " + groupString);
-        String query = avoidJoin ? SearchFields.PUBLIC_OBJECT + ":" + true : "";
-        if (groupString != null) {
+        return buildPermissionFilterQuery(avoidJoin, groupString);
+    }
+
+    private String buildPermissionFilterQuery(boolean avoidJoin, String permissionFilterGroups) {
+        String query = (avoidJoin&& !isAllGroups(permissionFilterGroups)) ? SearchFields.PUBLIC_OBJECT + ":" + true : "";
+        if (permissionFilterGroups != null && !isAllGroups(permissionFilterGroups)) {
             if (!query.isEmpty()) {
-                query = "(" + query + " OR " + "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + groupString + ")";
+                query = "(" + query + " OR " + "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + permissionFilterGroups + ")";
             } else {
-                query = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + groupString;
+                query = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + permissionFilterGroups;
             }
         }
         return query;
     }
-
+    
+    private boolean isAllGroups(String groups) {
+        return (groups!=null &&groups.equals(ALL_GROUPS));
+    }
 }
 

From 1aa2e6417ebaeb32b218b9345fa9950297880174 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 19 Jul 2024 17:22:56 -0400
Subject: [PATCH 008/137] drop extra line from merge

---
 src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java
index 935a003e0f7..adcc5825766 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java
@@ -235,5 +235,4 @@ public static String getGeoRadius(String userSuppliedGeoRadius) throws NumberFor
         return userSuppliedGeoRadius;
     }
 
-
 }

From d9488a767be75d612d26b998b27f3ad74501256e Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 19 Jul 2024 17:27:23 -0400
Subject: [PATCH 009/137] add logging to allow comparing old and new result

---
 .../edu/harvard/iq/dataverse/search/SearchServiceBean.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 6f07da8a96f..7ec77ccde09 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -1053,7 +1053,9 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
             groupString = groupList.get(0);
         }
         logger.fine("Groups: " + groupString);
-        return buildPermissionFilterQuery(avoidJoin, groupString);
+        String permissionQuery = buildPermissionFilterQuery(avoidJoin, groupString);
+        logger.fine("Permission Query: " + permissionQuery);
+        return permissionQuery;
     }
 
     private String buildPermissionFilterQuery(boolean avoidJoin, String permissionFilterGroups) {

From 52ca8a94aba382e69928e18455eeadf15a5568e3 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 21 Jun 2024 16:32:35 -0400
Subject: [PATCH 010/137] fix getProviderType

---
 .../pidproviders/doi/datacite/DataCiteDOIProvider.java         | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java
index cd765933796..b1dc49b7c6e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java
@@ -223,8 +223,7 @@ protected String getProviderKeyName() {
 
     @Override
     public String getProviderType() {
-        // TODO Auto-generated method stub
-        return null;
+        return TYPE;
     }
 
     public String getMdsUrl() {

From 1f87aa3cd9a1bbaebebc7e8df9cc4b6f246fbf08 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 21 Jun 2024 19:17:08 -0400
Subject: [PATCH 011/137] citation/case-insensitive DOI fixes

---
 .../iq/dataverse/api/MakeDataCountApi.java    | 23 +++++++++++++++----
 .../DatasetExternalCitationsServiceBean.java  | 12 ++++++----
 .../pidproviders/AbstractPidProvider.java     | 13 ++++++++---
 .../pidproviders/doi/AbstractDOIProvider.java | 22 +++++++++++++++++-
 4 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
index 1f2f1039327..306b863c9e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
@@ -19,6 +19,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Iterator;
 import java.util.List;
@@ -152,10 +155,17 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
             // DataCite wants "doi=", not "doi:".
             String authorityPlusIdentifier = persistentId.replaceFirst("doi:", "");
             // Request max page size and then loop to handle multiple pages
-            URL url = new URL(JvmSettings.DATACITE_REST_API_URL.lookup() +
+            URL url = null;
+            try {
+                url = new URI(JvmSettings.DATACITE_REST_API_URL.lookup(pidProvider.getId()) +
                               "/events?doi=" +
                               authorityPlusIdentifier +
-                              "&source=crossref&page[size]=1000");
+                              "&source=crossref&page[size]=1000").toURL();
+            } catch (URISyntaxException e) {
+                //Nominally this means a config error/ bad DATACITE_REST_API_URL for this provider
+                logger.warning("Unable to create URL for " + persistentId + ", pidProvider " + pidProvider.getId());
+                return error(Status.INTERNAL_SERVER_ERROR, "Unable to create DataCite URL to retrieve citations.");
+            }
             logger.fine("Retrieving Citations from " + url.toString());
             boolean nextPage = true;
             JsonArrayBuilder dataBuilder = Json.createArrayBuilder();
@@ -178,7 +188,12 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
                     dataBuilder.add(iter.next());
                 }
                 if (links.containsKey("next")) {
-                    url = new URL(links.getString("next"));
+                    try {
+                        url = new URI(links.getString("next")).toURL();
+                    } catch (URISyntaxException e) {
+                        logger.warning("Unable to create URL from DataCite response: " + links.getString("next"));
+                        return error(Status.INTERNAL_SERVER_ERROR, "Unable to retrieve all results from DataCite");
+                    }
                 } else {
                     nextPage = false;
                 }
@@ -187,7 +202,7 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
             JsonArray allData = dataBuilder.build();
             List<DatasetExternalCitations> datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData);
             /*
-             * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopuate them.
+             * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them.
              * As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation
              * as we've done for 'hasPart').
              * If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and
diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
index 50c24274bb2..9f930240edd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
@@ -7,6 +7,9 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
+import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -59,12 +62,11 @@ public List<DatasetExternalCitations> parseCitations(JsonArray citations) {
             if (inboundRelationships.contains(relationship)) {
                 Dataset localDs = null;
                 if (objectUri.contains("doi")) {
-                    String globalId = objectUri.replace("https://", "").replace("doi.org/", "doi:").toUpperCase().replace("DOI:", "doi:");
-                    localDs = datasetService.findByGlobalId(globalId);
+                    localDs = datasetService.findByGlobalId(objectUri);
                     exCit.setDataset(localDs);
                 }
                 exCit.setCitedByUrl(subjectUri);
-                
+
                 if (localDs != null && !exCit.getCitedByUrl().isEmpty()) {
                     datasetExternalCitations.add(exCit);
                 }
@@ -72,9 +74,9 @@ public List<DatasetExternalCitations> parseCitations(JsonArray citations) {
             if (outboundRelationships.contains(relationship)) {
                 Dataset localDs = null;
                 if (subjectUri.contains("doi")) {
-                    String globalId = subjectUri.replace("https://", "").replace("doi.org/", "doi:").toUpperCase().replace("DOI:", "doi:");
-                    localDs = datasetService.findByGlobalId(globalId);
+                    localDs = datasetService.findByGlobalId(subjectUri);
                     exCit.setDataset(localDs);
+
                 }
                 exCit.setCitedByUrl(objectUri);
                 
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
index f6d142aac96..d1a1331c348 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
@@ -36,9 +36,9 @@ public abstract class AbstractPidProvider implements PidProvider {
 
     private String datafilePidFormat = null;
 
-    private HashSet<String> managedSet;
+    protected HashSet<String> managedSet;
 
-    private HashSet<String> excludedSet;
+    protected HashSet<String> excludedSet;
 
     private String id;
     private String label;
@@ -313,10 +313,17 @@ protected GlobalId parsePersistentId(String protocol, String identifierString) {
     }
 
     public GlobalId parsePersistentId(String protocol, String authority, String identifier) {
+        return parsePersistentId(protocol, authority, identifier, false);
+    }
+    
+    public GlobalId parsePersistentId(String protocol, String authority, String identifier, boolean isCaseInsensitive) {
         logger.fine("Parsing: " + protocol + ":" + authority + getSeparator() + identifier + " in " + getId());
         if (!PidProvider.isValidGlobalId(protocol, authority, identifier)) {
             return null;
         }
+        if(isCaseInsensitive) {
+            identifier = identifier.toUpperCase();
+        }
         // Check authority/identifier if this is a provider that manages specific
         // identifiers
         // /is not one of the unmanaged providers that has null authority
@@ -333,7 +340,7 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden
             logger.fine("managed in " + getId() + ": " + getManagedSet().contains(cleanIdentifier));
             logger.fine("excluded from " + getId() + ": " + getExcludedSet().contains(cleanIdentifier));
 
-            if (!(((authority.equals(getAuthority()) && identifier.startsWith(getShoulder()))
+            if (!(((authority.equals(getAuthority()) && identifier.startsWith(getShoulder().toUpperCase()))
                     || getManagedSet().contains(cleanIdentifier)) && !getExcludedSet().contains(cleanIdentifier))) {
                 return null;
             }
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
index 43e34e74c59..532ea9e1fbe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.pidproviders.doi;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -26,6 +27,25 @@ public abstract class AbstractDOIProvider extends AbstractPidProvider {
 
     public AbstractDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList) {
         super(id, label, DOI_PROTOCOL, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList);
+        //Create case insensitive (converted toUpperCase) managedSet and excludedSet
+        HashSet<String> cleanManagedSet = new HashSet<String>();
+        for(String entry: managedSet) {
+            if(entry.startsWith(DOI_PROTOCOL)) {
+                cleanManagedSet.add(DOI_PROTOCOL + entry.substring(DOI_PROTOCOL.length()).toUpperCase());
+            } else {
+                logger.warning("Non-DOI found in managedSet of pidProvider id: " + getId() + ": " + entry + ". Entry is being dropped.");
+            }
+        }
+        managedSet = cleanManagedSet;
+        HashSet<String> cleanExcludedSet = new HashSet<String>();
+        for(String entry: excludedSet) {
+            if(entry.startsWith(DOI_PROTOCOL)) {
+                cleanExcludedSet.add(DOI_PROTOCOL + entry.substring(DOI_PROTOCOL.length()).toUpperCase());
+            } else {
+                logger.warning("Non-DOI found in excludedSet of pidProvider id: " + getId() + ": " + entry + ". Entry is being dropped.");
+            }
+        }
+        excludedSet = cleanExcludedSet;
     }
 
     //For Unmanged provider
@@ -67,7 +87,7 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden
         if (!DOI_PROTOCOL.equals(protocol)) {
             return null;
         }
-        return super.parsePersistentId(protocol, authority, identifier);
+        return super.parsePersistentId(protocol, authority, identifier, true);
     }
 
     public String getUrlPrefix() {

From b1cc199d5d47fb3ebc8426b6dbb524ad19622d7a Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 2 Jul 2024 14:32:56 -0400
Subject: [PATCH 012/137] avoid having an empty entry & log warning in
 managed/excluded lists

---
 .../iq/dataverse/pidproviders/AbstractPidProvider.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
index d1a1331c348..e26f7f3acc1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
@@ -60,8 +60,12 @@ protected AbstractPidProvider(String id, String label, String protocol, String a
         this.shoulder = shoulder;
         this.identifierGenerationStyle = identifierGenerationStyle;
         this.datafilePidFormat = datafilePidFormat;
-        this.managedSet = new HashSet<String>(Arrays.asList(managedList.split(",\\s")));
-        this.excludedSet = new HashSet<String>(Arrays.asList(excludedList.split(",\\s")));
+        if(!managedList.isEmpty()) {
+            this.managedSet = new HashSet<String>(Arrays.asList(managedList.split(",\\s")));
+        }
+        if(!excludedList.isEmpty()) {
+            this.excludedSet = new HashSet<String>(Arrays.asList(excludedList.split(",\\s")));
+        }
         if (logger.isLoggable(Level.FINE)) {
             Iterator<String> iter = managedSet.iterator();
             while (iter.hasNext()) {

From 08595ec646c48dd92781975a6e44c4b873ffd2ef Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 2 Jul 2024 14:47:10 -0400
Subject: [PATCH 013/137] avoid nulls

---
 .../iq/dataverse/pidproviders/AbstractPidProvider.java | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
index e26f7f3acc1..250eae7e5fc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
@@ -36,9 +36,9 @@ public abstract class AbstractPidProvider implements PidProvider {
 
     private String datafilePidFormat = null;
 
-    protected HashSet<String> managedSet;
+    protected HashSet<String> managedSet = new HashSet<String>();
 
-    protected HashSet<String> excludedSet;
+    protected HashSet<String> excludedSet = new HashSet<String>();
 
     private String id;
     private String label;
@@ -47,8 +47,6 @@ protected AbstractPidProvider(String id, String label, String protocol) {
         this.id = id;
         this.label = label;
         this.protocol = protocol;
-        this.managedSet = new HashSet<String>();
-        this.excludedSet = new HashSet<String>();
     }
 
     protected AbstractPidProvider(String id, String label, String protocol, String authority, String shoulder,
@@ -61,10 +59,10 @@ protected AbstractPidProvider(String id, String label, String protocol, String a
         this.identifierGenerationStyle = identifierGenerationStyle;
         this.datafilePidFormat = datafilePidFormat;
         if(!managedList.isEmpty()) {
-            this.managedSet = new HashSet<String>(Arrays.asList(managedList.split(",\\s")));
+            this.managedSet.addAll(Arrays.asList(managedList.split(",\\s")));
         }
         if(!excludedList.isEmpty()) {
-            this.excludedSet = new HashSet<String>(Arrays.asList(excludedList.split(",\\s")));
+            this.excludedSet.addAll(Arrays.asList(excludedList.split(",\\s")));
         }
         if (logger.isLoggable(Level.FINE)) {
             Iterator<String> iter = managedSet.iterator();

From e98a91f41adadcf1550852800db1fba8631c4e1b Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Sat, 20 Jul 2024 15:22:41 -0400
Subject: [PATCH 014/137] add is-supplement-to relation

---
 .../makedatacount/DatasetExternalCitationsServiceBean.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
index 9f930240edd..fa56432cc3c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
@@ -43,7 +43,8 @@ public class DatasetExternalCitationsServiceBean implements java.io.Serializable
           Arrays.asList(
           "cites",
           "references",
-          "supplements"));
+          "supplements",
+          "is-supplement-to"));
   static ArrayList<String> outboundRelationships = new ArrayList<String>( 
           Arrays.asList(
           "is-cited-by",

From 62fb0e83e5701a0683aabebaf10e360056b72297 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Sat, 20 Jul 2024 15:34:52 -0400
Subject: [PATCH 015/137] release note

---
 .../10708 - MDC Citation and DOI parsing improvements.md       | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/10708 - MDC Citation and DOI parsing improvements.md

diff --git a/doc/release-notes/10708 - MDC Citation and DOI parsing improvements.md b/doc/release-notes/10708 - MDC Citation and DOI parsing improvements.md
new file mode 100644
index 00000000000..1dcd293df77
--- /dev/null
+++ b/doc/release-notes/10708 - MDC Citation and DOI parsing improvements.md	
@@ -0,0 +1,3 @@
+MDC Citation retrieval with the PID settings has been fixed.
+DOI parsing in Dataverse is case insensitive, improving interaction with services that may change the case.
+Warnings related to managed/excluded PID lists for PID providers have been reduced

From 1cd9b7ce3d3e56bc2b0e1c2451215aaf5ff3430e Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Sat, 20 Jul 2024 15:43:37 -0400
Subject: [PATCH 016/137] test tweak for case insensitivity

---
 .../edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
index cffac741c78..b58c057c25c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
@@ -237,7 +237,8 @@ public void testDOIParsing() throws IOException {
         assertEquals(pid1String, pid3.asString());
         assertEquals("dc1", pid3.getProviderId());
         
-        String pid4String = "doi:10.5072/FK3ABCDEF";
+        //Also test case insensitive
+        String pid4String = "doi:10.5072/fk3ABCDEF";
         GlobalId pid4 = PidUtil.parseAsGlobalID(pid4String);
         assertEquals(pid4String, pid4.asString());
         assertEquals("dc2", pid4.getProviderId());

From 2ce7b54cdafbcb9f1dff4c9d889573e33b03983e Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Mon, 22 Jul 2024 15:52:34 +0200
Subject: [PATCH 017/137] Fix MultipleCompoundField for otherId

---
 .../harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index 85d4868605d..35d35316f73 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -210,7 +210,7 @@ private void processDDI(ImportType importType, XMLStreamReader xmlr, DatasetDTO
                 // study description section. we'll use the one we found in 
                 // the codeBook entry:
                 FieldDTO otherIdValue = FieldDTO.createPrimitiveFieldDTO("otherIdValue", codeBookLevelId);
-                FieldDTO otherId = FieldDTO.createCompoundFieldDTO("otherId", otherIdValue);
+                FieldDTO otherId = FieldDTO.createMultipleCompoundFieldDTO("otherId", otherIdValue);
                 citationBlock.getFields().add(otherId);
                 
             }

From bd500cfb2f01f6fc21e7f6fbb9a59d5219326bcc Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 3 Sep 2024 17:45:37 -0400
Subject: [PATCH 018/137] test fix

---
 .../edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
index b58c057c25c..b041b24dd29 100644
--- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
@@ -240,7 +240,9 @@ public void testDOIParsing() throws IOException {
         //Also test case insensitive
         String pid4String = "doi:10.5072/fk3ABCDEF";
         GlobalId pid4 = PidUtil.parseAsGlobalID(pid4String);
-        assertEquals(pid4String, pid4.asString());
+        // Lower case is recognized by converting to upper case internally, so we need to test vs. the upper case string
+        // I.e. we are verifying that the lower case string is parsed the same as the upper case string, buth give an internal upper case PID representation
+        assertEquals(pid4String.toUpperCase(), pid4.asString());
         assertEquals("dc2", pid4.getProviderId());
 
         String pid5String = "doi:10.5072/FK2ABCDEF";

From 7a0ee7fc208f12531ab00261eb78ecf156851c95 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 3 Sep 2024 18:00:07 -0400
Subject: [PATCH 019/137] keep doi: lower case

---
 .../edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
index b041b24dd29..9cb9e086ccd 100644
--- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java
@@ -240,9 +240,9 @@ public void testDOIParsing() throws IOException {
         //Also test case insensitive
         String pid4String = "doi:10.5072/fk3ABCDEF";
         GlobalId pid4 = PidUtil.parseAsGlobalID(pid4String);
-        // Lower case is recognized by converting to upper case internally, so we need to test vs. the upper case string
-        // I.e. we are verifying that the lower case string is parsed the same as the upper case string, buth give an internal upper case PID representation
-        assertEquals(pid4String.toUpperCase(), pid4.asString());
+        // Lower case is recognized by converting to upper case internally, so we need to test vs. the upper case identifier
+        // I.e. we are verifying that the lower case string is parsed the same as the upper case string, both give an internal upper case PID representation
+        assertEquals("doi:10.5072/FK3ABCDEF", pid4.asString());
         assertEquals("dc2", pid4.getProviderId());
 
         String pid5String = "doi:10.5072/FK2ABCDEF";

From 8a6744fc808eaa7abaf5ac28d6405333f4d7dc2e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 3 Sep 2024 22:27:42 +0000
Subject: [PATCH 020/137] Bump actions/download-artifact from 3 to 4.1.7 in
 /.github/workflows

Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4.1.7.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v3...v4.1.7)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/deploy_beta_testing.yml | 2 +-
 .github/workflows/maven_unit_test.yml     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml
index 028f0140cc9..efe3e0d8621 100644
--- a/.github/workflows/deploy_beta_testing.yml
+++ b/.github/workflows/deploy_beta_testing.yml
@@ -45,7 +45,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Download war artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4.1.7
         with:
           name: built-app
           path: ./
diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 4ad4798bc64..fe335f5f45a 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -105,7 +105,7 @@ jobs:
                   cache: maven
 
             # Get the build output from the unit test job
-            - uses: actions/download-artifact@v3
+            - uses: actions/download-artifact@v4.1.7
               with:
                   name: java-artifacts
             - run: |
@@ -137,7 +137,7 @@ jobs:
                 cache: maven
 
           # Get the build output from the integration test job
-          - uses: actions/download-artifact@v3
+          - uses: actions/download-artifact@v4.1.7
             with:
                 name: java-reportdir
           - run: tar -xvf java-reportdir.tar

From 03debc16cfc68e42a5bd4aad3076ed2dbadad94c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 9 Sep 2024 10:49:53 -0400
Subject: [PATCH 021/137] Some work-in-progress modifications of the Harvested
 imports and indexing process. #10734

---
 .../api/imports/ImportServiceBean.java        | 175 ++++++++++--------
 .../impl/AbstractCreateDatasetCommand.java    |  18 +-
 .../iq/dataverse/search/IndexServiceBean.java |   6 +-
 3 files changed, 118 insertions(+), 81 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index d2bba56f884..42d1a14bf1d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -20,6 +20,7 @@
 import edu.harvard.iq.dataverse.DataverseContact;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
 import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
 import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType;
@@ -268,29 +269,97 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
         }
 
         JsonObject obj = JsonUtil.getJsonObject(json);
-        //and call parse Json to read it into a dataset   
+        
+        String protocol = obj.getString("protocol", null);
+        String authority = obj.getString("authority", null);
+        String identifier = obj.getString("identifier",null);
+        
+        GlobalId globalId;
+        
+        // A Global ID is required:
+        // (meaning, we will fail with an exception if the imports above have 
+        // not managed to find an acceptable global identifier in the harvested 
+        // metadata)
+        
+        try { 
+            globalId = PidUtil.parseAsGlobalID(protocol, authority, identifier);
+        } catch (IllegalArgumentException iax) {
+            throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse can parse, skipping.");
+        }
+        
+        if (globalId == null) {
+            throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse recognizes, skipping.");
+        }
+        
+        String globalIdString = globalId.asString();
+        
+        if (StringUtils.isEmpty(globalIdString)) {
+            // @todo this check shouldn't be necessary, now that there's a null check above
+            throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse recognizes, skipping.");
+        }
+        
+        DatasetVersion harvestedVersion; 
+        
+        Dataset existingDataset = datasetService.findByGlobalId(globalIdString);
+        
+        
         try {
+            Dataset harvestedDataset = null;
+
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, datasetTypeService, harvestingClient);
             parser.setLenient(true);
-            Dataset ds = parser.parseDataset(obj);
 
-            // For ImportType.NEW, if the metadata contains a global identifier, and it's not a protocol
-            // we support, it should be rejected.
-            // (TODO: ! - add some way of keeping track of supported protocols!)
-            //if (ds.getGlobalId() != null && !ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, ""))) {
-            //    throw new ImportException("Could not register id " + ds.getGlobalId() + ", protocol not supported");
-            //}
-            ds.setOwner(owner);
-            ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());
+            if (existingDataset != null) {
+                // If this dataset already exists IN ANOTHER COLLECTION
+                // we are just going to skip it!
+                if (existingDataset.getOwner() != null && !owner.getId().equals(existingDataset.getOwner().getId())) {
+                    throw new ImportException("The dataset with the global id " + globalIdString + " already exists, in the dataverse " + existingDataset.getOwner().getAlias() + ", skipping.");
+                }
+                // And if we already have a dataset with this same id, in this same
+                // dataverse, but it is a LOCAL dataset (can happen!), we're going to 
+                // skip it also: 
+                if (!existingDataset.isHarvested()) {
+                    throw new ImportException("A LOCAL dataset with the global id " + globalIdString + " already exists in this dataverse; skipping.");
+                }
+                // For harvested datasets, there should always only be one version.
+                // We will replace the current version with the imported version.
+                // @todo or should we just destroy any extra versions quietly?
+                if (existingDataset.getVersions().size() != 1) {
+                    throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDataset.getVersions().size() + " versions");
+                }
+                
+                // ... do the magic - parse the version json, do the switcheroo ...
+                
+                harvestedDataset = existingDataset; // @todo!!
+                
+                harvestedVersion = parser.parseDatasetVersion(obj.getJsonObject("datasetVersion"));
+                
+            } else {
+                // Creating a new dataset from scratch:
+                
+                harvestedDataset = parser.parseDataset(obj);
+                harvestedDataset.setOwner(owner);
+                
+                harvestedDataset.setHarvestedFrom(harvestingClient);
+                harvestedDataset.setHarvestIdentifier(harvestIdentifier);
+                
+                harvestedVersion = harvestedDataset.getVersions().get(0);
+            }
+        
+            // Either a full new import, or an update of an existing harvested
+            // Dataset, perform some cleanup on the new version imported from the 
+            // parsed metadata:
+            
+            harvestedVersion.setDatasetFields(harvestedVersion.initDatasetFields());
 
-            if (ds.getVersions().get(0).getReleaseTime() == null) {
-                ds.getVersions().get(0).setReleaseTime(oaiDateStamp);
+            if (harvestedVersion.getReleaseTime() == null) {
+                harvestedVersion.setReleaseTime(oaiDateStamp);
             }
             
             // Check data against required contraints
-            List<ConstraintViolation<DatasetField>> violations = ds.getVersions().get(0).validateRequired();
+            List<ConstraintViolation<DatasetField>> violations = harvestedVersion.validateRequired();
             if (!violations.isEmpty()) {
-                // For migration and harvest, add NA for missing required values
+                // When harvesting, we add NA for missing required values:
                 for (ConstraintViolation<DatasetField> v : violations) {
                     DatasetField f =  v.getRootBean();
                     f.setSingleValue(DatasetField.NA_VALUE);
@@ -298,86 +367,42 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             }
 
             // Check data against validation constraints
-            // If we are migrating and "scrub migration data" is true we attempt to fix invalid data
-            // if the fix fails stop processing of this file by throwing exception
-            Set<ConstraintViolation> invalidViolations = ds.getVersions().get(0).validate();
-            ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
-            Validator validator = factory.getValidator();
+            // Similarly to how we handle missing required values (above), we 
+            // replace invalid values with NA when harvesting.
+            
+            Set<ConstraintViolation> invalidViolations = harvestedVersion.validate();
             if (!invalidViolations.isEmpty()) {
                 for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
                     DatasetFieldValue f = v.getRootBean();
-                    boolean fixed = false;
-                    boolean converted = false;
-                    // TODO: Is this scrubbing something we want to continue doing? 
-                    if (settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) {
-                        fixed = processMigrationValidationError(f, cleanupLog, metadataFile.getName());
-                        converted = true;
-                        if (fixed) {
-                            Set<ConstraintViolation<DatasetFieldValue>> scrubbedViolations = validator.validate(f);
-                            if (!scrubbedViolations.isEmpty()) {
-                                fixed = false;
-                            }
-                        }
-                    }
-                    if (!fixed) {
-                        String msg = "Data modified - File: " + metadataFile.getName() + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
-                                + "Invalid value:  '" + f.getValue() + "'" + " Converted Value:'" + DatasetField.NA_VALUE + "'";
-                        cleanupLog.println(msg);
-                        f.setValue(DatasetField.NA_VALUE);
 
-                    }
+                    String msg = "Invalid metadata: " + metadataFile.getName() + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
+                            + "Invalid value:  '" + f.getValue() + "'" + ", replaced with '" + DatasetField.NA_VALUE + "'";
+                    cleanupLog.println(msg);
+                    f.setValue(DatasetField.NA_VALUE);
                 }
             }
             
-            // A Global ID is required, in order for us to be able to harvest and import
-            // this dataset:
-            if (StringUtils.isEmpty(ds.getGlobalId().asString())) {
-                throw new ImportException("The harvested metadata record with the OAI server identifier "+harvestIdentifier+" does not contain a global unique identifier that we could recognize, skipping.");
-            }
-
-            ds.setHarvestedFrom(harvestingClient);
-            ds.setHarvestIdentifier(harvestIdentifier);
-            
-            Dataset existingDs = datasetService.findByGlobalId(ds.getGlobalId().asString());
-
-            if (existingDs != null) {
-                // If this dataset already exists IN ANOTHER DATAVERSE
-                // we are just going to skip it!
-                if (existingDs.getOwner() != null && !owner.getId().equals(existingDs.getOwner().getId())) {
-                    throw new ImportException("The dataset with the global id "+ds.getGlobalId().asString()+" already exists, in the dataverse "+existingDs.getOwner().getAlias()+", skipping.");
-                }
-                // And if we already have a dataset with this same id, in this same
-                // dataverse, but it is  LOCAL dataset (can happen!), we're going to 
-                // skip it also: 
-                if (!existingDs.isHarvested()) {
-                    throw new ImportException("A LOCAL dataset with the global id "+ds.getGlobalId().asString()+" already exists in this dataverse; skipping.");
-                }
-                // For harvested datasets, there should always only be one version.
-                // We will replace the current version with the imported version.
-                if (existingDs.getVersions().size() != 1) {
-                    throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDs.getVersions().size() + " versions");
-                }
+            if (existingDataset != null) {
                 // Purge all the SOLR documents associated with this client from the 
                 // index server: 
-                indexService.deleteHarvestedDocuments(existingDs);
+                indexService.deleteHarvestedDocuments(existingDataset);
                 // files from harvested datasets are removed unceremoniously, 
                 // directly in the database. no need to bother calling the 
                 // DeleteFileCommand on them.
-                for (DataFile harvestedFile : existingDs.getFiles()) {
+                for (DataFile harvestedFile : existingDataset.getFiles()) {
                     DataFile merged = em.merge(harvestedFile);
                     em.remove(merged);
                     harvestedFile = null; 
                 }
-                // TODO: 
-                // Verify what happens with the indexed files in SOLR? 
-                // are they going to be overwritten by the reindexing of the dataset?
-                existingDs.setFiles(null);
-                Dataset merged = em.merge(existingDs);
+                existingDataset.setFiles(null);
+                Dataset merged = em.merge(existingDataset);
                 // harvested datasets don't have physical files - so no need to worry about that.
                 engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest));
+                // @todo!
+                // UpdateHarvestedDatasetCommand() ?
+            } else {
+                importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(harvestedDataset, dataverseRequest));
             }
-            
-            importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(ds, dataverseRequest));
 
         } catch (JsonParseException | ImportException | CommandException ex) {
             logger.fine("Failed to import harvested dataset: " + ex.getClass() + ": " + ex.getMessage());
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java
index 7b7c5fd0e93..0567b737a58 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java
@@ -13,8 +13,10 @@
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.pidproviders.PidProvider;
 import static edu.harvard.iq.dataverse.util.StringUtil.isEmpty;
+import java.io.IOException;
 import java.util.Objects;
 import java.util.logging.Logger;
+import org.apache.solr.client.solrj.SolrServerException;
 
 /**;
  * An abstract base class for commands that creates {@link Dataset}s.
@@ -146,9 +148,19 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         
         //Use for code that requires database ids
         postDBFlush(theDataset, ctxt);
-        
-        ctxt.index().asyncIndexDataset(theDataset, true);
-                 
+
+        if (harvested) {
+            try {
+                ctxt.index().indexDataset(theDataset, true);
+            } catch (SolrServerException | IOException solrEx) {
+                logger.warning("Failed to index harvested dataset. " + solrEx.getMessage());
+            }
+        } else {
+            // The asynchronous version does not throw any exceptions, 
+            // logging them internally instead. 
+            ctxt.index().asyncIndexDataset(theDataset, true);
+        }
+               
         return theDataset;
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index fd769846490..a659976769a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -418,7 +418,7 @@ synchronized private static Dataset getNextToIndex(Long id, Dataset d) {
     public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
         try {
             acquirePermitFromSemaphore();
-            doAyncIndexDataset(dataset, doNormalSolrDocCleanUp);
+            doAsyncIndexDataset(dataset, doNormalSolrDocCleanUp);
         } catch (InterruptedException e) {
             String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString();
             failureLogText += "\r\n" + e.getLocalizedMessage();
@@ -428,7 +428,7 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
         }
     }
 
-    private void doAyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
+    private void doAsyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
         Long id = dataset.getId();
         Dataset next = getNextToIndex(id, dataset); // if there is an ongoing index job for this dataset, next is null (ongoing index job will reindex the newest version after current indexing finishes)
         while (next != null) {
@@ -449,7 +449,7 @@ public void asyncIndexDatasetList(List<Dataset> datasets, boolean doNormalSolrDo
         for(Dataset dataset : datasets) {
             try {
                 acquirePermitFromSemaphore();
-                doAyncIndexDataset(dataset, true);
+                doAsyncIndexDataset(dataset, true);
             } catch (InterruptedException e) {
                 String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString();
                 failureLogText += "\r\n" + e.getLocalizedMessage();

From e62fc55a1831999d6024d5a12c20cf8c81f172a4 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 10 Sep 2024 11:05:31 -0400
Subject: [PATCH 022/137] intermediate/unfinished state - work in progress
 #10734

---
 .../api/imports/ImportServiceBean.java        | 198 +++++++++++++++---
 .../iq/dataverse/search/IndexServiceBean.java |   5 +
 2 files changed, 171 insertions(+), 32 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index 42d1a14bf1d..a3147b68463 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -20,6 +20,7 @@
 import edu.harvard.iq.dataverse.DataverseContact;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
 import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
@@ -41,6 +42,7 @@
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import static edu.harvard.iq.dataverse.search.IndexServiceBean.solrDocIdentifierFile;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -71,6 +73,8 @@
 import jakarta.validation.Validation;
 import jakarta.validation.Validator;
 import jakarta.validation.ValidatorFactory;
+import java.util.HashMap;
+import java.util.Map;
 import javax.xml.stream.XMLStreamException;
 import org.apache.commons.lang3.StringUtils;
 
@@ -209,7 +213,7 @@ public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
     public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, String harvestIdentifier, String metadataFormat, File metadataFile, Date oaiDateStamp, PrintWriter cleanupLog) throws ImportException, IOException {
         if (harvestingClient == null || harvestingClient.getDataverse() == null) {
-            throw new ImportException("importHarvestedDataset called wiht a null harvestingClient, or an invalid harvestingClient.");
+            throw new ImportException("importHarvestedDataset called with a null harvestingClient, or an invalid harvestingClient.");
         }
         Dataverse owner = harvestingClient.getDataverse();
         Dataset importedDataset = null;
@@ -309,7 +313,19 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, datasetTypeService, harvestingClient);
             parser.setLenient(true);
 
-            if (existingDataset != null) {
+            if (existingDataset == null) {
+                // Creating a new dataset from scratch:
+                
+                harvestedDataset = parser.parseDataset(obj);
+                harvestedDataset.setOwner(owner);
+                
+                harvestedDataset.setHarvestedFrom(harvestingClient);
+                harvestedDataset.setHarvestIdentifier(harvestIdentifier);
+                
+                harvestedVersion = harvestedDataset.getVersions().get(0);
+            } else {
+                // We already have a dataset with this id in the database.
+                
                 // If this dataset already exists IN ANOTHER COLLECTION
                 // we are just going to skip it!
                 if (existingDataset.getOwner() != null && !owner.getId().equals(existingDataset.getOwner().getId())) {
@@ -327,23 +343,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 if (existingDataset.getVersions().size() != 1) {
                     throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDataset.getVersions().size() + " versions");
                 }
-                
-                // ... do the magic - parse the version json, do the switcheroo ...
-                
-                harvestedDataset = existingDataset; // @todo!!
-                
+                                
                 harvestedVersion = parser.parseDatasetVersion(obj.getJsonObject("datasetVersion"));
-                
-            } else {
-                // Creating a new dataset from scratch:
-                
-                harvestedDataset = parser.parseDataset(obj);
-                harvestedDataset.setOwner(owner);
-                
-                harvestedDataset.setHarvestedFrom(harvestingClient);
-                harvestedDataset.setHarvestIdentifier(harvestIdentifier);
-                
-                harvestedVersion = harvestedDataset.getVersions().get(0);
             }
         
             // Either a full new import, or an update of an existing harvested
@@ -370,6 +371,19 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             // Similarly to how we handle missing required values (above), we 
             // replace invalid values with NA when harvesting.
             
+            boolean sanitized = validateDatasetVersion(harvestedVersion, true, cleanupLog);
+            
+            // Note: this sanitizing approach, of replacing invalid values with 
+            // "NA" does not work with certain fields. For example, using it to 
+            // populate a GeoBox coordinate value will result in an invalid 
+            // field. So we will attempt to validate the santized version again,
+            // this time around, it will throw an exception if still invalid, so 
+            // we'll stop before proceeding any further. 
+            
+            if (sanitized) {
+                
+            }
+            
             Set<ConstraintViolation> invalidViolations = harvestedVersion.validate();
             if (!invalidViolations.isEmpty()) {
                 for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
@@ -379,27 +393,104 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                             + "Invalid value:  '" + f.getValue() + "'" + ", replaced with '" + DatasetField.NA_VALUE + "'";
                     cleanupLog.println(msg);
                     f.setValue(DatasetField.NA_VALUE);
+                    // Note: "NA" does not work with certain fields. For example, 
+                    // using it to populate a GeoBox coordinate value is going 
+                    // to result in an invalid field. @todo? - see below
                 }
             }
             
+            // @todo? - re-validate the version before we do anything else? 
+            // something along the lines of 
+            // if (this.validate) {validateOrDie(newVersion, false);}
+            // DatasetFieldUtil.tidyUpFields(newVersion.getDatasetFields(), true);
+            
             if (existingDataset != null) {
-                // Purge all the SOLR documents associated with this client from the 
-                // index server: 
-                indexService.deleteHarvestedDocuments(existingDataset);
-                // files from harvested datasets are removed unceremoniously, 
-                // directly in the database. no need to bother calling the 
-                // DeleteFileCommand on them.
-                for (DataFile harvestedFile : existingDataset.getFiles()) {
-                    DataFile merged = em.merge(harvestedFile);
-                    em.remove(merged);
-                    harvestedFile = null; 
+                // @todo
+                // ... do the magic - parse the version json, do the switcheroo ...
+                DatasetVersion existingVersion = existingDataset.getVersions().get(0);
+
+                Map<String, Integer> existingFilesIndex = new HashMap<>(); 
+                
+                for (int i = 0; i < existingDataset.getFiles().size(); i++) {
+                    String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
+                    if (storageIdentifier != null) {
+                        existingFilesIndex.put(storageIdentifier, i);
+                    }
+                }
+                
+                for (FileMetadata newFileMetadata : harvestedVersion.getFileMetadatas()) {
+                    // is it safe to assume that each new FileMetadata will be 
+                    // pointing to a non-null DataFile
+                    String location = newFileMetadata.getDataFile().getStorageIdentifier(); 
+                    if (location != null && existingFilesIndex.containsKey(location)) {
+                        newFileMetadata.getDataFile().setFileMetadatas(new ArrayList<>());
+                        
+                        int fileIndex = existingFilesIndex.get(location);
+                        newFileMetadata.setDataFile(existingDataset.getFiles().get(fileIndex));
+                        existingDataset.getFiles().get(fileIndex).getFileMetadatas().add(newFileMetadata);
+                        existingFilesIndex.remove(location);
+                    }
+                }
+                
+                List<String> solrIdsOfDocumentsToDelete = new ArrayList<>();
+                
+                // Go through the existing files and delete the ones that are 
+                // no longer present in the version that we have just harvesed:
+                for (FileMetadata oldFileMetadata : existingDataset.getVersions().get(0).getFileMetadatas()) {
+                    DataFile oldDataFile = oldFileMetadata.getDataFile();
+                    solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
+                    existingDataset.getFiles().remove(oldDataFile);
+                    // Files from harvested datasets are removed unceremoniously, 
+                    // directly in the database. No need to bother calling the 
+                    // DeleteFileCommand on them.
+                    em.remove(em.merge(oldDataFile));
+                    em.remove(em.merge(oldFileMetadata));
+                    oldDataFile = null;
+                    oldFileMetadata = null; 
+                }
+                
+                // purge all the SOLR documents associated with the files
+                // we have just deleted:
+                if (!solrIdsOfDocumentsToDelete.isEmpty()) {
+                    indexService.deleteHarvestedDocuments(solrIdsOfDocumentsToDelete);
+                }
+
+                // ... And now delete the existing version itself: 
+                
+                existingDataset.setVersions(new ArrayList<>());
+                em.remove(em.merge(existingVersion));
+                
+                // Now attach the newly-harvested version to the dataset:
+                existingDataset.getVersions().add(harvestedVersion);
+                harvestedVersion.setDataset(existingDataset);
+                
+                // ... There's one more thing to do - go through the new files, 
+                // that are not in the database yet, and make sure they are 
+                // attached to this existing dataset:
+                
+                for (FileMetadata newFileMetadata : harvestedVersion.getFileMetadatas()) {
+                    if (newFileMetadata.getDataFile().getId() == null) {
+                        existingDataset.getFiles().add(newFileMetadata.getDataFile());
+                        newFileMetadata.getDataFile().setOwner(existingDataset);
+                    }
                 }
-                existingDataset.setFiles(null);
-                Dataset merged = em.merge(existingDataset);
+                
+                em.persist(harvestedVersion);
+                
+                ///@todo remove for (DataFile harvestedFile : existingDataset.getFiles()) {
+                ///@todo remove     DataFile merged = em.merge(harvestedFile);
+                ///@todo remove     em.remove(merged);
+                ///@todo remove     harvestedFile = null; 
+                ///@todo remove }
+                ///@todo remove existingDataset.setFiles(null);
+                ///@todo remove Dataset merged = em.merge(existingDataset);
                 // harvested datasets don't have physical files - so no need to worry about that.
-                engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest));
+                ///@todo remove engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest));
                 // @todo!
-                // UpdateHarvestedDatasetCommand() ?
+                // UpdateHarvestedDatasetCommand() ? (later on)
+                importedDataset = em.merge(existingDataset);
+                //@todo reindex 
+
             } else {
                 importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(harvestedDataset, dataverseRequest));
             }
@@ -421,6 +512,49 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
         }
         return importedDataset;
     }
+    /**
+     * Shortcut method for validating AND attempting to sanitize a DatasetVersion
+     * @param version
+     * @param cleanupLog - any invalid values and their replacements are logged there
+     * @return true if any invalid values were encountered and sanitized
+     * @throws ImportException (although it should never happen in this mode)
+     */
+    private boolean validateAndSanitizeVersionMetadata(DatasetVersion version, PrintWriter cleanupLog) throws ImportException {
+        return validateVersionMetadata(version, true, cleanupLog);
+    }
+    
+    private void validateVersionMetadata(DatasetVersion version, PrintWriter log) throws ImportException {
+        validateVersionMetadata(version, false, log);
+    } 
+    
+    private boolean validateVersionMetadata(DatasetVersion version, boolean sanitize, PrintWriter cleanupLog) throws ImportException {
+        boolean fixed = false;
+        Set<ConstraintViolation> invalidViolations = version.validate();
+        if (!invalidViolations.isEmpty()) {
+            for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
+                DatasetFieldValue f = v.getRootBean();
+
+                String msg = "Invalid metadata field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
+                        + "Invalid value:  '" + f.getValue() + "'";
+                if (sanitize) {
+                    msg += ", replaced with '" + DatasetField.NA_VALUE + "'";
+                    f.setValue(DatasetField.NA_VALUE);
+                    fixed = true;
+                }
+                cleanupLog.println(msg);
+
+                // Note: "NA" does not work with certain fields. For example, 
+                // using it to populate a GeoBox coordinate value is going 
+                // to result in an invalid field. So we'll need to validate the 
+                // version again after the first, sanitizing pass and see if it 
+                // helped or not.
+            }
+            if (!sanitize) {
+                throw new ImportException("Version was still failing validation after the first attempt to sanitize the invalid values.");
+            }
+        }
+        return fixed;
+    }
 
     public JsonObject ddiToJson(String xmlToParse) throws ImportException, XMLStreamException {
         DatasetDTO dsDTO = importDDIService.doImport(ImportType.IMPORT, xmlToParse);
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index a659976769a..834ea5f1ed9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -2398,6 +2398,11 @@ public void deleteHarvestedDocuments(Dataset harvestedDataset) {
             solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + datafile.getId());
         }
 
+        deleteHarvestedDocuments(solrIdsOfDocumentsToDelete);
+    }
+    
+    public void deleteHarvestedDocuments(List<String> solrIdsOfDocumentsToDelete) {
+
         logger.fine("attempting to delete the following documents from the index: " + StringUtils.join(solrIdsOfDocumentsToDelete, ","));
         IndexResponse resultOfAttemptToDeleteDocuments = solrIndexService.deleteMultipleSolrIds(solrIdsOfDocumentsToDelete);
         logger.fine("result of attempt to delete harvested documents: " + resultOfAttemptToDeleteDocuments + "\n");

From 3fab4c8c823afbca7b28e74031aa9fa2ceb65bf0 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 10 Sep 2024 13:54:36 -0400
Subject: [PATCH 023/137] incremental. #10734

---
 .../api/imports/ImportServiceBean.java        | 130 ++++++++------
 .../impl/UpdateHarvestedDatasetCommand.java   | 159 ++++++++++++++++++
 2 files changed, 238 insertions(+), 51 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index a3147b68463..b3c65ceefcc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -33,6 +33,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.CreateHarvestedDatasetCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.UpdateHarvestedDatasetCommand;
 import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -43,6 +44,7 @@
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
 import static edu.harvard.iq.dataverse.search.IndexServiceBean.solrDocIdentifierFile;
+import edu.harvard.iq.dataverse.util.DatasetFieldUtil;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -366,25 +368,29 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                     f.setSingleValue(DatasetField.NA_VALUE);
                 }
             }
-
+            
+            // @todo? - is this the right place to call tidyUpFields()? 
+            // usually it is called within the body of the create/update commands.
+            DatasetFieldUtil.tidyUpFields(harvestedVersion.getDatasetFields(), true);
+            
             // Check data against validation constraints
             // Similarly to how we handle missing required values (above), we 
             // replace invalid values with NA when harvesting.
             
-            boolean sanitized = validateDatasetVersion(harvestedVersion, true, cleanupLog);
+            boolean sanitized = validateAndSanitizeVersionMetadata(harvestedVersion, cleanupLog);
             
             // Note: this sanitizing approach, of replacing invalid values with 
             // "NA" does not work with certain fields. For example, using it to 
             // populate a GeoBox coordinate value will result in an invalid 
-            // field. So we will attempt to validate the santized version again,
-            // this time around, it will throw an exception if still invalid, so 
-            // we'll stop before proceeding any further. 
+            // field. So we will attempt to re-validate the santized version.
+            // This time around, it will throw an exception if still invalid, so 
+            // that we'll stop before proceeding any further: 
             
             if (sanitized) {
-                
+                validateVersionMetadata(harvestedVersion, cleanupLog);
             }
             
-            Set<ConstraintViolation> invalidViolations = harvestedVersion.validate();
+            /*Set<ConstraintViolation> invalidViolations = harvestedVersion.validate();
             if (!invalidViolations.isEmpty()) {
                 for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
                     DatasetFieldValue f = v.getRootBean();
@@ -397,7 +403,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                     // using it to populate a GeoBox coordinate value is going 
                     // to result in an invalid field. @todo? - see below
                 }
-            }
+            }*/
             
             // @todo? - re-validate the version before we do anything else? 
             // something along the lines of 
@@ -407,6 +413,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             if (existingDataset != null) {
                 // @todo
                 // ... do the magic - parse the version json, do the switcheroo ...
+                /*
                 DatasetVersion existingVersion = existingDataset.getVersions().get(0);
 
                 Map<String, Integer> existingFilesIndex = new HashMap<>(); 
@@ -490,6 +497,9 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 // UpdateHarvestedDatasetCommand() ? (later on)
                 importedDataset = em.merge(existingDataset);
                 //@todo reindex 
+                */
+                
+                importedDataset = engineSvc.submit(new UpdateHarvestedDatasetCommand(existingDataset, harvestedVersion, dataverseRequest));
 
             } else {
                 importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(harvestedDataset, dataverseRequest));
@@ -512,49 +522,6 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
         }
         return importedDataset;
     }
-    /**
-     * Shortcut method for validating AND attempting to sanitize a DatasetVersion
-     * @param version
-     * @param cleanupLog - any invalid values and their replacements are logged there
-     * @return true if any invalid values were encountered and sanitized
-     * @throws ImportException (although it should never happen in this mode)
-     */
-    private boolean validateAndSanitizeVersionMetadata(DatasetVersion version, PrintWriter cleanupLog) throws ImportException {
-        return validateVersionMetadata(version, true, cleanupLog);
-    }
-    
-    private void validateVersionMetadata(DatasetVersion version, PrintWriter log) throws ImportException {
-        validateVersionMetadata(version, false, log);
-    } 
-    
-    private boolean validateVersionMetadata(DatasetVersion version, boolean sanitize, PrintWriter cleanupLog) throws ImportException {
-        boolean fixed = false;
-        Set<ConstraintViolation> invalidViolations = version.validate();
-        if (!invalidViolations.isEmpty()) {
-            for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
-                DatasetFieldValue f = v.getRootBean();
-
-                String msg = "Invalid metadata field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
-                        + "Invalid value:  '" + f.getValue() + "'";
-                if (sanitize) {
-                    msg += ", replaced with '" + DatasetField.NA_VALUE + "'";
-                    f.setValue(DatasetField.NA_VALUE);
-                    fixed = true;
-                }
-                cleanupLog.println(msg);
-
-                // Note: "NA" does not work with certain fields. For example, 
-                // using it to populate a GeoBox coordinate value is going 
-                // to result in an invalid field. So we'll need to validate the 
-                // version again after the first, sanitizing pass and see if it 
-                // helped or not.
-            }
-            if (!sanitize) {
-                throw new ImportException("Version was still failing validation after the first attempt to sanitize the invalid values.");
-            }
-        }
-        return fixed;
-    }
 
     public JsonObject ddiToJson(String xmlToParse) throws ImportException, XMLStreamException {
         DatasetDTO dsDTO = importDDIService.doImport(ImportType.IMPORT, xmlToParse);
@@ -855,6 +822,67 @@ private String convertInvalidDateString(String inString){
         return null;
     }
 
+    /**
+     * A shortcut method for validating AND attempting to sanitize a DatasetVersion
+     * @param version
+     * @param cleanupLog - any invalid values and their replacements are logged there
+     * @return true if any invalid values were encountered and sanitized
+     * @throws ImportException (although it should never happen in this mode)
+     */
+    private boolean validateAndSanitizeVersionMetadata(DatasetVersion version, PrintWriter cleanupLog) throws ImportException {
+        return validateVersionMetadata(version, true, cleanupLog);
+    }
+    
+    /**
+     * A shortcut method for validating a DatasetVersion; will throw an exception 
+     * if invalid, without attempting to sanitize the invalid values. 
+     * @param version
+     * @param log - will log the invalid fields encountered there 
+     * @throws ImportException 
+     */
+    private void validateVersionMetadata(DatasetVersion version, PrintWriter log) throws ImportException {
+        validateVersionMetadata(version, false, log);
+    } 
+    
+    /**
+     * Validate the metadata fields of a newly-created version, and depending on 
+     * the "sanitize" flag supplied, may or may not attempt to sanitize the supplied
+     * values by replacing them with "NA"s. 
+     * @param version
+     * @param sanitize - boolean indicating whether to attempt to fix invalid values
+     * @param cleanupLog - to log any invalid values encountered will be logged
+     * @return - true if any invalid values have been replaced  
+     * @throws ImportException 
+     */
+    private boolean validateVersionMetadata(DatasetVersion version, boolean sanitize, PrintWriter cleanupLog) throws ImportException {
+        boolean fixed = false;
+        Set<ConstraintViolation> invalidViolations = version.validate();
+        if (!invalidViolations.isEmpty()) {
+            for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
+                DatasetFieldValue f = v.getRootBean();
+
+                String msg = "Invalid metadata field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
+                        + "Invalid value:  '" + f.getValue() + "'";
+                if (sanitize) {
+                    msg += ", replaced with '" + DatasetField.NA_VALUE + "'";
+                    f.setValue(DatasetField.NA_VALUE);
+                    fixed = true;
+                }
+                cleanupLog.println(msg);
+
+                // Note: "NA" does not work with certain fields. For example, 
+                // using it to populate a GeoBox coordinate value is going 
+                // to result in an invalid field. So we'll need to validate the 
+                // version again after the first, sanitizing pass and see if it 
+                // helped or not.
+            }
+            if (!sanitize) {
+                throw new ImportException("Version was still failing validation after the first attempt to sanitize the invalid values.");
+            }
+        }
+        return fixed;
+    }
+
 
     private static class MyCustomFormatter extends Formatter {
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
new file mode 100644
index 00000000000..d28950e4d9d
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
@@ -0,0 +1,159 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.FileMetadata;
+import static edu.harvard.iq.dataverse.search.IndexServiceBean.solrDocIdentifierFile;
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.solr.client.solrj.SolrServerException;
+
+/**
+ *
+ * @author landreev
+ * 
+ * Much simplified version of UpdateDatasetVersionCommand, 
+ * but with some extra twists. 
+ */
+@RequiredPermissions(Permission.EditDataset)
+public class UpdateHarvestedDatasetCommand extends AbstractDatasetCommand<Dataset> {
+
+    private static final Logger logger = Logger.getLogger(UpdateHarvestedDatasetCommand.class.getCanonicalName());
+    private final DatasetVersion newHarvestedVersion;
+    final private boolean validateLenient = true;
+
+    public UpdateHarvestedDatasetCommand(Dataset theDataset, DatasetVersion newHarvestedVersion, DataverseRequest aRequest) {
+        super(aRequest, theDataset);
+        this.newHarvestedVersion = newHarvestedVersion;
+    }
+
+    public boolean isValidateLenient() {
+        return validateLenient;
+    }
+
+    @Override
+    public Dataset execute(CommandContext ctxt) throws CommandException {
+        
+        // ... do the magic - parse the version json, do the switcheroo ...
+        Dataset existingDataset = getDataset();
+
+        if (existingDataset == null
+                || existingDataset.getId() == null
+                || !existingDataset.isHarvested()
+                || existingDataset.getVersions().size() != 1) {
+            throw new IllegalCommandException("The command can only be called on an existing harvested dataset with only 1 version", this);
+        }
+        DatasetVersion existingVersion = existingDataset.getVersions().get(0);
+
+        if (newHarvestedVersion == null || newHarvestedVersion.getId() != null) {
+            throw new IllegalCommandException("The command can only be called with a newly-harvested, not yet saved DatasetVersion supplied", this);
+        }
+        
+        Map<String, Integer> existingFilesIndex = new HashMap<>();
+
+        for (int i = 0; i < existingDataset.getFiles().size(); i++) {
+            String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
+            if (storageIdentifier != null) {
+                existingFilesIndex.put(storageIdentifier, i);
+            }
+        }
+                
+        for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
+            // is it safe to assume that each new FileMetadata will be 
+            // pointing to a non-null DataFile here?
+            String location = newFileMetadata.getDataFile().getStorageIdentifier();
+            if (location != null && existingFilesIndex.containsKey(location)) {
+                newFileMetadata.getDataFile().setFileMetadatas(new ArrayList<>());
+
+                int fileIndex = existingFilesIndex.get(location);
+                newFileMetadata.setDataFile(existingDataset.getFiles().get(fileIndex));
+                existingDataset.getFiles().get(fileIndex).getFileMetadatas().add(newFileMetadata);
+                existingFilesIndex.remove(location);
+            }
+        }
+        // @todo check that the newly-harvested DataFiles have the same checksums
+        // and mime types etc. These values are supposed to be immutable, normally, 
+        // but who knows - they may have fixed something invalid on the other end
+        // @todo check if there's anything special that needs to be done for things
+        // like file categories
+                
+        List<String> solrIdsOfDocumentsToDelete = new ArrayList<>();
+
+        // Go through the existing files and delete the ones that are 
+        // no longer present in the version that we have just harvesed:
+        for (FileMetadata oldFileMetadata : existingDataset.getVersions().get(0).getFileMetadatas()) {
+            DataFile oldDataFile = oldFileMetadata.getDataFile();
+            solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
+            existingDataset.getFiles().remove(oldDataFile);
+            // Files from harvested datasets are removed unceremoniously, 
+            // directly in the database. No need to bother calling the 
+            // DeleteFileCommand on them.
+            ctxt.em().remove(ctxt.em().merge(oldDataFile));
+            ctxt.em().remove(ctxt.em().merge(oldFileMetadata));
+            oldDataFile = null;
+            oldFileMetadata = null;
+        }
+                
+        // purge all the SOLR documents associated with the files
+        // we have just deleted:
+        if (!solrIdsOfDocumentsToDelete.isEmpty()) {
+            ctxt.index().deleteHarvestedDocuments(solrIdsOfDocumentsToDelete);
+        }
+
+        // ... And now delete the existing version itself: 
+        existingDataset.setVersions(new ArrayList<>());
+        ctxt.em().remove(ctxt.em().merge(existingVersion));
+
+        // Now attach the newly-harvested version to the dataset:
+        existingDataset.getVersions().add(newHarvestedVersion);
+        newHarvestedVersion.setDataset(existingDataset);
+                
+        // ... There's one more thing to do - go through the new files, 
+        // that are not in the database yet, and make sure they are 
+        // attached to this existing dataset:
+        for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
+            if (newFileMetadata.getDataFile().getId() == null) {
+                existingDataset.getFiles().add(newFileMetadata.getDataFile());
+                newFileMetadata.getDataFile().setOwner(existingDataset);
+            }
+        }
+                
+        ctxt.em().persist(newHarvestedVersion);
+                
+        Dataset savedDataset = ctxt.em().merge(existingDataset);
+        ctxt.em().flush();
+        
+        //@todo reindex 
+        
+        return savedDataset;
+    }
+
+    @Override
+    public boolean onSuccess(CommandContext ctxt, Object r) {
+        boolean retVal = true;
+        Dataset d = (Dataset) r;
+        
+        try {
+            // Note that we index harvested datasets synchronously:
+            ctxt.index().indexDataset(d, true);
+        } catch (SolrServerException|IOException solrServerEx) {
+            logger.log(Level.WARNING, "Exception while trying to index the updated Harvested dataset " + d.getGlobalId().asString(), solrServerEx.getMessage());
+            retVal = false;
+        }
+        
+        return retVal;
+    }
+}

From bd08b698859db5c5662f8a742504832023a632d7 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 11 Sep 2024 14:37:34 -0400
Subject: [PATCH 024/137] Largely finalized versions of the refactored
 harvesting classes. #10734

---
 .../api/imports/ImportServiceBean.java        | 142 ++++--------------
 .../impl/UpdateHarvestedDatasetCommand.java   |  82 +++++++---
 2 files changed, 90 insertions(+), 134 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index b3c65ceefcc..8c7ff5acf6d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -300,7 +300,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
         String globalIdString = globalId.asString();
         
         if (StringUtils.isEmpty(globalIdString)) {
-            // @todo this check shouldn't be necessary, now that there's a null check above
+            // @todo this check may not be necessary, now that there's a null check above
             throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse recognizes, skipping.");
         }
         
@@ -347,6 +347,11 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 }
                                 
                 harvestedVersion = parser.parseDatasetVersion(obj.getJsonObject("datasetVersion"));
+                Dataset tempDataset = createTemporaryHarvestedDataset(harvestedVersion);
+                // Temporarily attach the temporary dataset to the parent Collection: 
+                // (this will be needed for the purposes of looking up the configured
+                // metadatablocks and such)
+                tempDataset.setOwner(owner);
             }
         
             // Either a full new import, or an update of an existing harvested
@@ -362,15 +367,16 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             // Check data against required contraints
             List<ConstraintViolation<DatasetField>> violations = harvestedVersion.validateRequired();
             if (!violations.isEmpty()) {
-                // When harvesting, we add NA for missing required values:
+                // ... and fill the missing required values with "NA"s:
                 for (ConstraintViolation<DatasetField> v : violations) {
                     DatasetField f =  v.getRootBean();
                     f.setSingleValue(DatasetField.NA_VALUE);
                 }
             }
             
-            // @todo? - is this the right place to call tidyUpFields()? 
-            // usually it is called within the body of the create/update commands.
+            // is this the right place to call tidyUpFields()? 
+            // usually it is called within the body of the create/update commands
+            // later on.
             DatasetFieldUtil.tidyUpFields(harvestedVersion.getDatasetFields(), true);
             
             // Check data against validation constraints
@@ -379,7 +385,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             
             boolean sanitized = validateAndSanitizeVersionMetadata(harvestedVersion, cleanupLog);
             
-            // Note: this sanitizing approach, of replacing invalid values with 
+            // Note: our sanitizing approach, of replacing invalid values with 
             // "NA" does not work with certain fields. For example, using it to 
             // populate a GeoBox coordinate value will result in an invalid 
             // field. So we will attempt to re-validate the santized version.
@@ -390,117 +396,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 validateVersionMetadata(harvestedVersion, cleanupLog);
             }
             
-            /*Set<ConstraintViolation> invalidViolations = harvestedVersion.validate();
-            if (!invalidViolations.isEmpty()) {
-                for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
-                    DatasetFieldValue f = v.getRootBean();
-
-                    String msg = "Invalid metadata: " + metadataFile.getName() + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
-                            + "Invalid value:  '" + f.getValue() + "'" + ", replaced with '" + DatasetField.NA_VALUE + "'";
-                    cleanupLog.println(msg);
-                    f.setValue(DatasetField.NA_VALUE);
-                    // Note: "NA" does not work with certain fields. For example, 
-                    // using it to populate a GeoBox coordinate value is going 
-                    // to result in an invalid field. @todo? - see below
-                }
-            }*/
-            
-            // @todo? - re-validate the version before we do anything else? 
-            // something along the lines of 
-            // if (this.validate) {validateOrDie(newVersion, false);}
-            // DatasetFieldUtil.tidyUpFields(newVersion.getDatasetFields(), true);
-            
             if (existingDataset != null) {
-                // @todo
-                // ... do the magic - parse the version json, do the switcheroo ...
-                /*
-                DatasetVersion existingVersion = existingDataset.getVersions().get(0);
-
-                Map<String, Integer> existingFilesIndex = new HashMap<>(); 
-                
-                for (int i = 0; i < existingDataset.getFiles().size(); i++) {
-                    String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
-                    if (storageIdentifier != null) {
-                        existingFilesIndex.put(storageIdentifier, i);
-                    }
-                }
-                
-                for (FileMetadata newFileMetadata : harvestedVersion.getFileMetadatas()) {
-                    // is it safe to assume that each new FileMetadata will be 
-                    // pointing to a non-null DataFile
-                    String location = newFileMetadata.getDataFile().getStorageIdentifier(); 
-                    if (location != null && existingFilesIndex.containsKey(location)) {
-                        newFileMetadata.getDataFile().setFileMetadatas(new ArrayList<>());
-                        
-                        int fileIndex = existingFilesIndex.get(location);
-                        newFileMetadata.setDataFile(existingDataset.getFiles().get(fileIndex));
-                        existingDataset.getFiles().get(fileIndex).getFileMetadatas().add(newFileMetadata);
-                        existingFilesIndex.remove(location);
-                    }
-                }
-                
-                List<String> solrIdsOfDocumentsToDelete = new ArrayList<>();
-                
-                // Go through the existing files and delete the ones that are 
-                // no longer present in the version that we have just harvesed:
-                for (FileMetadata oldFileMetadata : existingDataset.getVersions().get(0).getFileMetadatas()) {
-                    DataFile oldDataFile = oldFileMetadata.getDataFile();
-                    solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
-                    existingDataset.getFiles().remove(oldDataFile);
-                    // Files from harvested datasets are removed unceremoniously, 
-                    // directly in the database. No need to bother calling the 
-                    // DeleteFileCommand on them.
-                    em.remove(em.merge(oldDataFile));
-                    em.remove(em.merge(oldFileMetadata));
-                    oldDataFile = null;
-                    oldFileMetadata = null; 
-                }
-                
-                // purge all the SOLR documents associated with the files
-                // we have just deleted:
-                if (!solrIdsOfDocumentsToDelete.isEmpty()) {
-                    indexService.deleteHarvestedDocuments(solrIdsOfDocumentsToDelete);
-                }
-
-                // ... And now delete the existing version itself: 
-                
-                existingDataset.setVersions(new ArrayList<>());
-                em.remove(em.merge(existingVersion));
-                
-                // Now attach the newly-harvested version to the dataset:
-                existingDataset.getVersions().add(harvestedVersion);
-                harvestedVersion.setDataset(existingDataset);
-                
-                // ... There's one more thing to do - go through the new files, 
-                // that are not in the database yet, and make sure they are 
-                // attached to this existing dataset:
-                
-                for (FileMetadata newFileMetadata : harvestedVersion.getFileMetadatas()) {
-                    if (newFileMetadata.getDataFile().getId() == null) {
-                        existingDataset.getFiles().add(newFileMetadata.getDataFile());
-                        newFileMetadata.getDataFile().setOwner(existingDataset);
-                    }
-                }
-                
-                em.persist(harvestedVersion);
-                
-                ///@todo remove for (DataFile harvestedFile : existingDataset.getFiles()) {
-                ///@todo remove     DataFile merged = em.merge(harvestedFile);
-                ///@todo remove     em.remove(merged);
-                ///@todo remove     harvestedFile = null; 
-                ///@todo remove }
-                ///@todo remove existingDataset.setFiles(null);
-                ///@todo remove Dataset merged = em.merge(existingDataset);
-                // harvested datasets don't have physical files - so no need to worry about that.
-                ///@todo remove engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest));
-                // @todo!
-                // UpdateHarvestedDatasetCommand() ? (later on)
-                importedDataset = em.merge(existingDataset);
-                //@todo reindex 
-                */
-                
                 importedDataset = engineSvc.submit(new UpdateHarvestedDatasetCommand(existingDataset, harvestedVersion, dataverseRequest));
-
             } else {
                 importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(harvestedDataset, dataverseRequest));
             }
@@ -883,6 +780,23 @@ private boolean validateVersionMetadata(DatasetVersion version, boolean sanitize
         return fixed;
     }
 
+    /**
+     * Helper method that creates a throwaway Harvested Dataset to temporarily 
+     * attach the newly-harvested version to. We need this when, instead of 
+     * importing a brand-new harvested dataset from scratch, we are planning to
+     * attempt to update an already existing dataset harvested from the same 
+     * archival location.
+     * @param harvestedVersion - a newly created Version imported from harvested metadata
+     * @return - a temporary dataset to which the new version has been attached
+     */
+    private Dataset createTemporaryHarvestedDataset(DatasetVersion harvestedVersion) {
+        Dataset tempDataset = new Dataset();
+        harvestedVersion.setDataset(tempDataset);
+        tempDataset.setVersions(new ArrayList<>(1));
+        tempDataset.getVersions().add(harvestedVersion);
+        
+        return tempDataset;
+    }
 
     private static class MyCustomFormatter extends Formatter {
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
index d28950e4d9d..9d7461b9b30 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
@@ -26,7 +26,10 @@
  * @author landreev
  * 
  * Much simplified version of UpdateDatasetVersionCommand, 
- * but with some extra twists. 
+ * but with some extra twists. The goal is to avoid creating new Dataset and 
+ * DataFile objects, and to instead preserve the database ids of the re-harvested 
+ * datasets and files, whenever possible. This in turn allows us to avoid deleting
+ * and rebuilding from scratch the Solr documents for these objects. 
  */
 @RequiredPermissions(Permission.EditDataset)
 public class UpdateHarvestedDatasetCommand extends AbstractDatasetCommand<Dataset> {
@@ -47,7 +50,6 @@ public boolean isValidateLenient() {
     @Override
     public Dataset execute(CommandContext ctxt) throws CommandException {
         
-        // ... do the magic - parse the version json, do the switcheroo ...
         Dataset existingDataset = getDataset();
 
         if (existingDataset == null
@@ -62,15 +64,30 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
             throw new IllegalCommandException("The command can only be called with a newly-harvested, not yet saved DatasetVersion supplied", this);
         }
         
+        newHarvestedVersion.setCreateTime(getTimestamp());
+        newHarvestedVersion.setLastUpdateTime(getTimestamp());
+
+        
         Map<String, Integer> existingFilesIndex = new HashMap<>();
 
+        /* 
+        Create a map of the files that are currently part of this existing
+        harvested dataset. We assume that a harvested file can be uniquely 
+        defined by its storageidentifier. Which, in the case of a datafile
+        harvested from another Dataverse should be its data access api url. 
+        */
         for (int i = 0; i < existingDataset.getFiles().size(); i++) {
             String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
             if (storageIdentifier != null) {
                 existingFilesIndex.put(storageIdentifier, i);
             }
         }
-                
+        
+        /*
+        Go through the files in the newly-harvested version and check if any of 
+        them are the files (potentially new/updated versions thereof) of the files
+        we already have, harvested previously from the same archive location. 
+        */
         for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
             // is it safe to assume that each new FileMetadata will be 
             // pointing to a non-null DataFile here?
@@ -79,14 +96,32 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                 newFileMetadata.getDataFile().setFileMetadatas(new ArrayList<>());
 
                 int fileIndex = existingFilesIndex.get(location);
+                
+                // Make sure to update the existing DataFiles that we are going 
+                // to keep in case their newly-harvested versions have different 
+                // checksums, mime types etc. These values are supposed to be 
+                // immutable, normally - but who knows, errors happen, the source 
+                // Dataverse may have had to fix these in their database to 
+                // correct a data integrity issue (for ex.):
+                existingDataset.getFiles().get(fileIndex).setContentType(newFileMetadata.getDataFile().getContentType());
+                existingDataset.getFiles().get(fileIndex).setFilesize(newFileMetadata.getDataFile().getFilesize());
+                existingDataset.getFiles().get(fileIndex).setChecksumType(newFileMetadata.getDataFile().getChecksumType());
+                existingDataset.getFiles().get(fileIndex).setChecksumValue(newFileMetadata.getDataFile().getChecksumValue());
+                
+                // Point the newly-harvested filemetadata to the existing datafile:
                 newFileMetadata.setDataFile(existingDataset.getFiles().get(fileIndex));
+                
+                // Make sure this new FileMetadata is the only one attached to this existing file:
+                existingDataset.getFiles().get(fileIndex).setFileMetadatas(new ArrayList<>(1));
                 existingDataset.getFiles().get(fileIndex).getFileMetadatas().add(newFileMetadata);
+                // (we don't want any cascade relationships left between this existing
+                // dataset and its version, since we are going to attemp to delete it).
+                
+                // Drop the file from the index map:
                 existingFilesIndex.remove(location);
             }
         }
-        // @todo check that the newly-harvested DataFiles have the same checksums
-        // and mime types etc. These values are supposed to be immutable, normally, 
-        // but who knows - they may have fixed something invalid on the other end
+        
         // @todo check if there's anything special that needs to be done for things
         // like file categories
                 
@@ -96,15 +131,20 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         // no longer present in the version that we have just harvesed:
         for (FileMetadata oldFileMetadata : existingDataset.getVersions().get(0).getFileMetadatas()) {
             DataFile oldDataFile = oldFileMetadata.getDataFile();
-            solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
-            existingDataset.getFiles().remove(oldDataFile);
-            // Files from harvested datasets are removed unceremoniously, 
-            // directly in the database. No need to bother calling the 
-            // DeleteFileCommand on them.
-            ctxt.em().remove(ctxt.em().merge(oldDataFile));
-            ctxt.em().remove(ctxt.em().merge(oldFileMetadata));
-            oldDataFile = null;
-            oldFileMetadata = null;
+            String location = oldDataFile.getStorageIdentifier();
+            // Is it still in the existing files map? - that means it is no longer
+            // present in the newly-harvested version
+            if (location != null && existingFilesIndex.containsKey(location)) {
+                solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
+                existingDataset.getFiles().remove(oldDataFile);
+                // Files from harvested datasets are removed unceremoniously, 
+                // directly in the database. No need to bother calling the 
+                // DeleteFileCommand on them. We'll just need to remember to purge
+                // them from Solr as well (right below)
+                ctxt.em().remove(ctxt.em().merge(oldDataFile));
+                // (no need to explicitly remove the oldFileMetadata; it will be 
+                // removed with the entire old version is deleted)
+            }
         }
                 
         // purge all the SOLR documents associated with the files
@@ -115,7 +155,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
 
         // ... And now delete the existing version itself: 
         existingDataset.setVersions(new ArrayList<>());
-        ctxt.em().remove(ctxt.em().merge(existingVersion));
+        existingVersion.setDataset(null);
+        
+        existingVersion = ctxt.em().merge(existingVersion);
+        ctxt.em().remove(existingVersion);
 
         // Now attach the newly-harvested version to the dataset:
         existingDataset.getVersions().add(newHarvestedVersion);
@@ -123,7 +166,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                 
         // ... There's one more thing to do - go through the new files, 
         // that are not in the database yet, and make sure they are 
-        // attached to this existing dataset:
+        // attached to this existing dataset, instead of the dummy temp 
+        // dataset into which they were originally imported:
         for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
             if (newFileMetadata.getDataFile().getId() == null) {
                 existingDataset.getFiles().add(newFileMetadata.getDataFile());
@@ -135,9 +179,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                 
         Dataset savedDataset = ctxt.em().merge(existingDataset);
         ctxt.em().flush();
-        
-        //@todo reindex 
-        
+                
         return savedDataset;
     }
 

From e02a5845ca02e86ec042411f32513c3c35536f5a Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 11 Sep 2024 15:24:33 -0400
Subject: [PATCH 025/137] a typo + some unused imports #10734

---
 .../harvard/iq/dataverse/api/imports/ImportServiceBean.java  | 5 -----
 .../engine/command/impl/UpdateHarvestedDatasetCommand.java   | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index 8c7ff5acf6d..acfb696b71d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -7,7 +7,6 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetField;
 import edu.harvard.iq.dataverse.DatasetFieldConstant;
@@ -20,7 +19,6 @@
 import edu.harvard.iq.dataverse.DataverseContact;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
-import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
 import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
@@ -43,7 +41,6 @@
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
-import static edu.harvard.iq.dataverse.search.IndexServiceBean.solrDocIdentifierFile;
 import edu.harvard.iq.dataverse.util.DatasetFieldUtil;
 
 import java.io.File;
@@ -75,8 +72,6 @@
 import jakarta.validation.Validation;
 import jakarta.validation.Validator;
 import jakarta.validation.ValidatorFactory;
-import java.util.HashMap;
-import java.util.Map;
 import javax.xml.stream.XMLStreamException;
 import org.apache.commons.lang3.StringUtils;
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
index 9d7461b9b30..c93983befa5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
@@ -115,7 +115,7 @@ them are the files (potentially new/updated versions thereof) of the files
                 existingDataset.getFiles().get(fileIndex).setFileMetadatas(new ArrayList<>(1));
                 existingDataset.getFiles().get(fileIndex).getFileMetadatas().add(newFileMetadata);
                 // (we don't want any cascade relationships left between this existing
-                // dataset and its version, since we are going to attemp to delete it).
+                // dataset and this version, since we are going to attemp to delete it).
                 
                 // Drop the file from the index map:
                 existingFilesIndex.remove(location);

From 30266ddcafc96127430e1a22dcee722920ae76cd Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 11 Sep 2024 15:28:30 -0400
Subject: [PATCH 026/137] comment language #10734

---
 .../engine/command/impl/UpdateHarvestedDatasetCommand.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
index c93983befa5..2380d7309ee 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
@@ -74,7 +74,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         Create a map of the files that are currently part of this existing
         harvested dataset. We assume that a harvested file can be uniquely 
         defined by its storageidentifier. Which, in the case of a datafile
-        harvested from another Dataverse should be its data access api url. 
+        harvested from another Dataverse should be its data access api url.
         */
         for (int i = 0; i < existingDataset.getFiles().size(); i++) {
             String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
@@ -85,8 +85,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         
         /*
         Go through the files in the newly-harvested version and check if any of 
-        them are the files (potentially new/updated versions thereof) of the files
-        we already have, harvested previously from the same archive location. 
+        them are (potentially new/updated) versions of files that we already
+        have, harvested previously from the same archive location. 
         */
         for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
             // is it safe to assume that each new FileMetadata will be 

From 9291554e7e83faa74cbee9da955f4aa87cd070b3 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 11 Sep 2024 17:18:10 -0400
Subject: [PATCH 027/137] minor/cosmetic #10734

---
 .../api/imports/ImportServiceBean.java          | 10 +++++-----
 .../impl/UpdateHarvestedDatasetCommand.java     | 17 +++++++++--------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index acfb696b71d..78bf9af99fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -328,9 +328,9 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 if (existingDataset.getOwner() != null && !owner.getId().equals(existingDataset.getOwner().getId())) {
                     throw new ImportException("The dataset with the global id " + globalIdString + " already exists, in the dataverse " + existingDataset.getOwner().getAlias() + ", skipping.");
                 }
-                // And if we already have a dataset with this same id, in this same
-                // dataverse, but it is a LOCAL dataset (can happen!), we're going to 
-                // skip it also: 
+                // And if we already have a dataset with this same global id at 
+                // this Dataverse instance, but it is a LOCAL dataset (can happen!), 
+                // we're going to skip it also: 
                 if (!existingDataset.isHarvested()) {
                     throw new ImportException("A LOCAL dataset with the global id " + globalIdString + " already exists in this dataverse; skipping.");
                 }
@@ -359,7 +359,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 harvestedVersion.setReleaseTime(oaiDateStamp);
             }
             
-            // Check data against required contraints
+            // Check data against required constraints
             List<ConstraintViolation<DatasetField>> violations = harvestedVersion.validateRequired();
             if (!violations.isEmpty()) {
                 // ... and fill the missing required values with "NA"s:
@@ -457,7 +457,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o
             ds.setOwner(owner);
             ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());
 
-            // Check data against required contraints
+            // Check data against required constraints
             List<ConstraintViolation<DatasetField>> violations = ds.getVersions().get(0).validateRequired();
             if (!violations.isEmpty()) {
                 if ( importType.equals(ImportType.HARVEST) ) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
index 2380d7309ee..09563686299 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
@@ -11,6 +11,7 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.FileMetadata;
 import static edu.harvard.iq.dataverse.search.IndexServiceBean.solrDocIdentifierFile;
+import edu.harvard.iq.dataverse.util.StringUtil;
 import java.io.IOException;
 
 import java.util.ArrayList;
@@ -78,7 +79,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         */
         for (int i = 0; i < existingDataset.getFiles().size(); i++) {
             String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
-            if (storageIdentifier != null) {
+            if (!StringUtil.isEmpty(storageIdentifier)) {
                 existingFilesIndex.put(storageIdentifier, i);
             }
         }
@@ -91,11 +92,11 @@ them are (potentially new/updated) versions of files that we already
         for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
             // is it safe to assume that each new FileMetadata will be 
             // pointing to a non-null DataFile here?
-            String location = newFileMetadata.getDataFile().getStorageIdentifier();
-            if (location != null && existingFilesIndex.containsKey(location)) {
+            String storageIdentifier = newFileMetadata.getDataFile().getStorageIdentifier();
+            if (!StringUtil.isEmpty(storageIdentifier) && existingFilesIndex.containsKey(storageIdentifier)) {
                 newFileMetadata.getDataFile().setFileMetadatas(new ArrayList<>());
 
-                int fileIndex = existingFilesIndex.get(location);
+                int fileIndex = existingFilesIndex.get(storageIdentifier);
                 
                 // Make sure to update the existing DataFiles that we are going 
                 // to keep in case their newly-harvested versions have different 
@@ -118,11 +119,11 @@ them are (potentially new/updated) versions of files that we already
                 // dataset and this version, since we are going to attemp to delete it).
                 
                 // Drop the file from the index map:
-                existingFilesIndex.remove(location);
+                existingFilesIndex.remove(storageIdentifier);
             }
         }
         
-        // @todo check if there's anything special that needs to be done for things
+        // @todo? check if there's anything special that needs to be done for things
         // like file categories
                 
         List<String> solrIdsOfDocumentsToDelete = new ArrayList<>();
@@ -131,10 +132,10 @@ them are (potentially new/updated) versions of files that we already
         // no longer present in the version that we have just harvesed:
         for (FileMetadata oldFileMetadata : existingDataset.getVersions().get(0).getFileMetadatas()) {
             DataFile oldDataFile = oldFileMetadata.getDataFile();
-            String location = oldDataFile.getStorageIdentifier();
+            String storageIdentifier = oldDataFile.getStorageIdentifier();
             // Is it still in the existing files map? - that means it is no longer
             // present in the newly-harvested version
-            if (location != null && existingFilesIndex.containsKey(location)) {
+            if (StringUtil.isEmpty(storageIdentifier) || existingFilesIndex.containsKey(storageIdentifier)) {
                 solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
                 existingDataset.getFiles().remove(oldDataFile);
                 // Files from harvested datasets are removed unceremoniously, 

From 8db46f5e832eec2a161a30a1ee6639c687843e67 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Sep 2024 11:52:43 +0100
Subject: [PATCH 028/137] Added: optional query param returnExpiration to
 token/recreate endpoint

---
 .../edu/harvard/iq/dataverse/api/Users.java   | 16 +++++-------
 .../edu/harvard/iq/dataverse/api/UsersIT.java | 26 +++++++++++++++----
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 11 +++++---
 3 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java
index 1f5430340c2..c1a7c95dbff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java
@@ -24,13 +24,7 @@
 import jakarta.ejb.Stateless;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObjectBuilder;
-import jakarta.ws.rs.BadRequestException;
-import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.*;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.MediaType;
@@ -157,7 +151,7 @@ public Response getTokenExpirationDate() {
     @Path("token/recreate")
     @AuthRequired
     @POST
-    public Response recreateToken(@Context ContainerRequestContext crc) {
+    public Response recreateToken(@Context ContainerRequestContext crc, @QueryParam("returnExpiration") boolean returnExpiration) {
         User u = getRequestUser(crc);
 
         AuthenticatedUser au;        
@@ -174,8 +168,12 @@ public Response recreateToken(@Context ContainerRequestContext crc) {
         ApiToken newToken = authSvc.generateApiTokenForUser(au);
         authSvc.save(newToken);
 
-        return ok("New token for " + au.getUserIdentifier() + " is " + newToken.getTokenString());
+        String message = "New token for " + au.getUserIdentifier() + " is " + newToken.getTokenString();
+        if (returnExpiration) {
+            message += " and expires on " + newToken.getExpireTime();
+        }
 
+        return ok(message);
     }
     
     @GET
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java
index 0189ffd6e58..1003c1a990c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java
@@ -23,6 +23,8 @@
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.contains;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.hamcrest.CoreMatchers;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
@@ -371,23 +373,33 @@ public void testAPITokenEndpoints() {
                 .body("data.message", containsString(userApiToken))
                 .body("data.message", containsString("expires on"));
 
+        // Recreate given a bad API token
         Response recreateToken = UtilIT.recreateToken("BAD-Token-blah-89234");
         recreateToken.prettyPrint();
         recreateToken.then().assertThat()
                 .statusCode(UNAUTHORIZED.getStatusCode());
 
+        // Recreate given a valid API token
         recreateToken = UtilIT.recreateToken(userApiToken);
         recreateToken.prettyPrint();
         recreateToken.then().assertThat()
                 .statusCode(OK.getStatusCode())
-                .body("data.message", containsString("New token for"));
+                .body("data.message", containsString("New token for"))
+                .body("data.message", CoreMatchers.not(containsString("and expires on")));
 
+        // Recreate given a valid API token and returning expiration
         createUser = UtilIT.createRandomUser();
-        createUser.prettyPrint();
-        assertEquals(200, createUser.getStatusCode());
+        assertEquals(OK.getStatusCode(), createUser.getStatusCode());
+
+        userApiToken =  UtilIT.getApiTokenFromResponse(createUser);
+
+        recreateToken = UtilIT.recreateToken(userApiToken, true);
+        recreateToken.prettyPrint();
+        recreateToken.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.message", containsString("New token for"))
+                .body("data.message", containsString("and expires on"));
 
-        String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser);
-        
         /*
         Add tests for Private URL
         */
@@ -418,6 +430,10 @@ public void testAPITokenEndpoints() {
         getExpiration.then().assertThat()
                 .statusCode(NOT_FOUND.getStatusCode());
 
+        createUser = UtilIT.createRandomUser();
+        assertEquals(OK.getStatusCode(), createUser.getStatusCode());
+
+        String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser);
 
         Response deleteToken = UtilIT.deleteToken(userApiTokenForDelete);
         deleteToken.prettyPrint();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4e20e8e4c33..8aa89bd0dd2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2813,10 +2813,15 @@ static Response getTokenExpiration( String apiToken) {
         return response;
     }
     
-    static Response recreateToken( String apiToken) {
+    static Response recreateToken(String apiToken) {
+        return recreateToken(apiToken, false);
+    }
+
+    static Response recreateToken(String apiToken, boolean returnExpiration) {
         Response response = given()
-            .header(API_TOKEN_HTTP_HEADER, apiToken)
-            .post("api/users/token/recreate");
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .queryParam("returnExpiration", returnExpiration)
+                .post("api/users/token/recreate");
         return response;
     }
 

From bdf626b15381fd61921440fd6e5a673e20447184 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Sep 2024 14:18:21 +0100
Subject: [PATCH 029/137] Added: release notes and docs for #10857

---
 .../10857-add-expiration-date-to-recreate-token-api.md      | 1 +
 doc/sphinx-guides/source/api/native-api.rst                 | 6 ++++++
 2 files changed, 7 insertions(+)
 create mode 100644 doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md

diff --git a/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md b/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md
new file mode 100644
index 00000000000..b450867c630
--- /dev/null
+++ b/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md
@@ -0,0 +1 @@
+An optional query parameter called 'returnExpiration' has been added to the 'users/token/recreate' endpoint, which, if set to true, returns the expiration time in the response message.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 117aceb141d..370cd889cfc 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -4412,6 +4412,12 @@ In order to obtain a new token use::
 
 	curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate"
 
+This endpoint by default will return a response message indicating the user identifier and the new token.
+
+To also include the expiration time in the response message, the query parameter ``returnExpiration`` must be set to true::
+
+	curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate?returnExpiration=true"
+
 Delete a Token
 ~~~~~~~~~~~~~~
 

From 7568cd05c4a82331b5539491a76c547cb00a4e93 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 18 Sep 2024 22:44:03 -0400
Subject: [PATCH 030/137] Some cleanup/streamlining #10734

---
 .../api/imports/ImportServiceBean.java        | 42 ++++++++-----------
 1 file changed, 17 insertions(+), 25 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index 78bf9af99fa..73d47b18b9a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -303,9 +303,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
         
         Dataset existingDataset = datasetService.findByGlobalId(globalIdString);
         
-        
         try {
-            Dataset harvestedDataset = null;
+            Dataset harvestedDataset;
 
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, datasetTypeService, harvestingClient);
             parser.setLenient(true);
@@ -314,7 +313,6 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 // Creating a new dataset from scratch:
                 
                 harvestedDataset = parser.parseDataset(obj);
-                harvestedDataset.setOwner(owner);
                 
                 harvestedDataset.setHarvestedFrom(harvestingClient);
                 harvestedDataset.setHarvestIdentifier(harvestIdentifier);
@@ -322,6 +320,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 harvestedVersion = harvestedDataset.getVersions().get(0);
             } else {
                 // We already have a dataset with this id in the database.
+                // Let's check a few things before we go any further with it: 
                 
                 // If this dataset already exists IN ANOTHER COLLECTION
                 // we are just going to skip it!
@@ -335,20 +334,22 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                     throw new ImportException("A LOCAL dataset with the global id " + globalIdString + " already exists in this dataverse; skipping.");
                 }
                 // For harvested datasets, there should always only be one version.
-                // We will replace the current version with the imported version.
-                // @todo or should we just destroy any extra versions quietly?
                 if (existingDataset.getVersions().size() != 1) {
                     throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDataset.getVersions().size() + " versions");
                 }
-                                
+                
+                // We will attempt to import the new version, and replace the 
+                // current, already existing version with it.                
                 harvestedVersion = parser.parseDatasetVersion(obj.getJsonObject("datasetVersion"));
-                Dataset tempDataset = createTemporaryHarvestedDataset(harvestedVersion);
-                // Temporarily attach the temporary dataset to the parent Collection: 
-                // (this will be needed for the purposes of looking up the configured
-                // metadatablocks and such)
-                tempDataset.setOwner(owner);
+                
+                // For the purposes of validation, the version needs to be attached
+                // to a non-null dataset. We will reate a throwaway temporary 
+                // dataset for this:
+                harvestedDataset = createTemporaryHarvestedDataset(harvestedVersion);
             }
         
+            harvestedDataset.setOwner(owner);
+            
             // Either a full new import, or an update of an existing harvested
             // Dataset, perform some cleanup on the new version imported from the 
             // parsed metadata:
@@ -359,28 +360,19 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 harvestedVersion.setReleaseTime(oaiDateStamp);
             }
             
-            // Check data against required constraints
-            List<ConstraintViolation<DatasetField>> violations = harvestedVersion.validateRequired();
-            if (!violations.isEmpty()) {
-                // ... and fill the missing required values with "NA"s:
-                for (ConstraintViolation<DatasetField> v : violations) {
-                    DatasetField f =  v.getRootBean();
-                    f.setSingleValue(DatasetField.NA_VALUE);
-                }
-            }
-            
             // is this the right place to call tidyUpFields()? 
             // usually it is called within the body of the create/update commands
             // later on.
             DatasetFieldUtil.tidyUpFields(harvestedVersion.getDatasetFields(), true);
             
-            // Check data against validation constraints
-            // Similarly to how we handle missing required values (above), we 
-            // replace invalid values with NA when harvesting.
+            // Check data against validation constraints. 
+            // Make an attempt to sanitize any invalid fields encountered - 
+            // missing required fields or invalid values, by filling the values 
+            // with NAs.
             
             boolean sanitized = validateAndSanitizeVersionMetadata(harvestedVersion, cleanupLog);
             
-            // Note: our sanitizing approach, of replacing invalid values with 
+            // Note: this sanitizing approach, of replacing invalid values with 
             // "NA" does not work with certain fields. For example, using it to 
             // populate a GeoBox coordinate value will result in an invalid 
             // field. So we will attempt to re-validate the santized version.

From 740b18daa001aa1d11ca0e6aa13132b4f442cfa9 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Thu, 19 Sep 2024 14:44:00 -0400
Subject: [PATCH 031/137] include validDataLocation in cached value/query

---
 src/main/java/edu/harvard/iq/dataverse/api/Metrics.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
index 452e5df9f9a..f36c514859e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
@@ -206,12 +206,13 @@ public Response getDatasetsTimeSeriest(@Context Request req, @Context UriInfo ur
             return error(BAD_REQUEST, ia.getLocalizedMessage());
         }
         String metricName = "datasets";
-        JsonArray jsonArray = MetricsUtil.stringToJsonArray(metricsSvc.returnUnexpiredCacheAllTime(metricName, null, d));
+        String validDataLocation = MetricsUtil.validateDataLocationStringType(dataLocation);
+        JsonArray jsonArray = MetricsUtil.stringToJsonArray(metricsSvc.returnUnexpiredCacheAllTime(metricName, validDataLocation, d));
 
         if (null == jsonArray) { // run query and save
 
-            jsonArray = metricsSvc.getDatasetsTimeSeries(uriInfo, dataLocation, d);
-            metricsSvc.save(new Metric(metricName, null, null, d, jsonArray.toString()));
+            jsonArray = metricsSvc.getDatasetsTimeSeries(uriInfo, validDataLocation, d);
+            metricsSvc.save(new Metric(metricName, null, validDataLocation, d, jsonArray.toString()));
         }
         MediaType requestedType = getVariant(req, MediaType.valueOf(FileUtil.MIME_TYPE_CSV), MediaType.APPLICATION_JSON_TYPE);
         if ((requestedType != null) && (requestedType.equals(MediaType.APPLICATION_JSON_TYPE))) {

From efb90050c69b6fcb37a0221694c9bfab873c4901 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Thu, 19 Sep 2024 14:44:49 -0400
Subject: [PATCH 032/137] fix queries w.r.t. minorversion ordering

---
 .../dataverse/metrics/MetricsServiceBean.java | 39 +++++++------------
 1 file changed, 13 insertions(+), 26 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index a74474efa15..5bdbeac031d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -168,25 +168,12 @@ public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
             }
         }
 
-        // Note that this SQL line in the code below:
-        // datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))
-        // behaves somewhat counter-intuitively if the versionnumber and/or
-        // minorversionnumber is/are NULL - it results in an empty string
-        // (NOT the string "{dataset_id}:", in other words). Some harvested
-        // versions do not have version numbers (only the ones harvested from
-        // other Dataverses!) It works fine
-        // for our purposes below, because we are simply counting the selected
-        // lines - i.e. we don't care if some of these lines are empty.
-        // But do not use this notation if you need the values returned to
-        // meaningfully identify the datasets!
-
-
         Query query = em.createNativeQuery(
 
 
                 "select count(*)\n"
                         + "from (\n"
-                        + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
+                        + "select DISTINCT ON (datasetversion.dataset_id) datasetversion.dataset_id \n"
                         + "from datasetversion\n"
                         + "join dataset on dataset.id = datasetversion.dataset_id\n"
                         + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
@@ -194,7 +181,7 @@ public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
                         + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n ")
                         + "and \n"
                         + dataLocationLine // be careful about adding more and statements after this line.
-                        + "group by dataset_id \n"
+                        + " order by datasetversion.dataset_id, datasetversion.versionnumber desc, datasetversion.minorversionnumber desc\n"
             +") sub_temp"
         );
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -207,15 +194,15 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
         // A published local datasets may have more than one released version!
         // So that's why we have to jump through some extra hoops below
         // in order to select the latest one:
-        String originClause = "(datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in\n" +
+        String originClause = "(datasetversion.id in\n" +
                 "(\n" +
-                "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n" +
+                "select DISTINCT ON (datasetversion.dataset_id) datasetversion.id\n" +
                 "       from datasetversion\n" +
                 "       join dataset on dataset.id = datasetversion.dataset_id\n" +
                 "       where versionstate='RELEASED'\n" +
                 "       	     and dataset.harvestingclient_id is null\n" +
                 "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n" +
-                "       group by dataset_id\n" +
+                "       order by datasetversion.dataset_id, datasetversion.versionnumber desc, datasetversion.minorversionnumber desc\n" +
                 "))\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
@@ -273,7 +260,7 @@ public long datasetsPastDays(int days, String dataLocation, Dataverse d) {
         Query query = em.createNativeQuery(
                 "select count(*)\n"
                         + "from (\n"
-                        + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max\n"
+                        + "select DISTINCT ON (datasetversion.dataset_id) datasetversion.id\n"
                         + "from datasetversion\n"
                         + "join dataset on dataset.id = datasetversion.dataset_id\n"
                         + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
@@ -281,7 +268,7 @@ public long datasetsPastDays(int days, String dataLocation, Dataverse d) {
                         + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                         + "and \n"
                         + dataLocationLine // be careful about adding more and statements after this line.
-                        + "group by dataset_id \n"
+                        + " order by datasetversion.dataset_id, datasetversion.versionnumber desc, datasetversion.minorversionnumber desc \n"
             +") sub_temp"
         );
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -322,9 +309,9 @@ public long filesToMonth(String yyyymm, Dataverse d) {
                 + "select count(*)\n"
                 + "from filemetadata\n"
                 + "join datasetversion on datasetversion.id = filemetadata.datasetversion_id\n"
-                + "where datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in \n"
+                + "where datasetversion.id in \n"
                 + "(\n"
-                + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
+                + "select DISTINCT ON (datasetversion.dataset_id) datasetversion.id \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
                 + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
@@ -332,7 +319,7 @@ public long filesToMonth(String yyyymm, Dataverse d) {
                 + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                 + "and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
                 + "and dataset.harvestingclient_id is null\n"
-                + "group by dataset_id \n"
+                + "order by datasetversion.dataset_id, datasetversion.versionnumber desc, datasetversion.minorversionnumber \n"
                 + ");"
         );
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -345,9 +332,9 @@ public long filesPastDays(int days, Dataverse d) {
                 + "select count(*)\n"
                 + "from filemetadata\n"
                 + "join datasetversion on datasetversion.id = filemetadata.datasetversion_id\n"
-                + "where datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in \n"
+                + "where datasetversion.id in \n"
                 + "(\n"
-                + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
+                + "select DISTINCT ON (datasetversion.dataset_id) datasetversion.id \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
                 + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
@@ -355,7 +342,7 @@ public long filesPastDays(int days, Dataverse d) {
                 + "and releasetime > current_date - interval '" + days + "' day\n"
                 + ((d == null) ? "" : "AND dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                 + "and dataset.harvestingclient_id is null\n"
-                + "group by dataset_id \n"
+                + "order by datasetversion.dataset_id, datasetversion.versionnumber desc, datasetversion.minorversionnumber desc \n"
                 + ");"
         );
         logger.log(Level.FINE, "Metric query: {0}", query);

From 35c9b5abcbec7aaaa5f93209f16da7740dd6147d Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Thu, 19 Sep 2024 15:12:04 -0400
Subject: [PATCH 033/137] release note

---
 doc/release-notes/10379-MetricsBugsFixes.md | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/release-notes/10379-MetricsBugsFixes.md

diff --git a/doc/release-notes/10379-MetricsBugsFixes.md b/doc/release-notes/10379-MetricsBugsFixes.md
new file mode 100644
index 00000000000..0ebc6d99f0b
--- /dev/null
+++ b/doc/release-notes/10379-MetricsBugsFixes.md
@@ -0,0 +1,10 @@
+
+### Metrics API Bug fixes
+
+Two bugs in the Metrics API have been fixed:
+
+- The /datasets and /datasets/byMonth endpoints could report incorrect values if/when they have been called using the dataLocation parameter (which allows getting metrics for local, remote (harvested), or all datasets) as the metrics cache was not storing different values for these cases. 
+
+- Metrics endpoints who's calculation relied on finding the latest published datasetversion were incorrect if/when the minor version number was > 9. 
+
+When deploying the new release, the [/api/admin/clearMetricsCache](https://guides.dataverse.org/en/latest/api/native-api.html#metrics) API should be called to remove old cached values that may be incorrect.
\ No newline at end of file

From 0a83bc374ab9c4cccdf7d8e7b9fbe74ce4948848 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 23 Sep 2024 11:22:08 -0400
Subject: [PATCH 034/137] typo in comment

---
 .../edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index 73d47b18b9a..66f48bfb872 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -343,7 +343,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 harvestedVersion = parser.parseDatasetVersion(obj.getJsonObject("datasetVersion"));
                 
                 // For the purposes of validation, the version needs to be attached
-                // to a non-null dataset. We will reate a throwaway temporary 
+                // to a non-null dataset. We will create a throwaway temporary 
                 // dataset for this:
                 harvestedDataset = createTemporaryHarvestedDataset(harvestedVersion);
             }

From 110fbe10dd57d6c881a31ceb602c78470aed2ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= <pkiraly@gwdg.de>
Date: Tue, 20 Jun 2023 17:35:58 +0200
Subject: [PATCH 035/137] #9650 improve the linked dataverse listing API

---
 .../java/edu/harvard/iq/dataverse/api/Datasets.java    | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index f26b6d67edf..1700c4fbcf5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2076,10 +2076,16 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id")
             List<Dataverse> dvsThatLinkToThisDatasetId = dataverseSvc.findDataversesThatLinkToThisDatasetId(datasetId);
             JsonArrayBuilder dataversesThatLinkToThisDatasetIdBuilder = Json.createArrayBuilder();
             for (Dataverse dataverse : dvsThatLinkToThisDatasetId) {
-                dataversesThatLinkToThisDatasetIdBuilder.add(dataverse.getAlias() + " (id " + dataverse.getId() + ")");
+                JsonObjectBuilder entry = Json.createObjectBuilder();
+                entry.add("id", dataverse.getId());
+                entry.add("alias", dataverse.getAlias());
+                entry.add("displayName", dataverse.getDisplayName());
+                dataversesThatLinkToThisDatasetIdBuilder.add(entry);
             }
             JsonObjectBuilder response = Json.createObjectBuilder();
-            response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder);
+            response.add("id", datasetId);
+            response.add("identifier", dataset.getIdentifier());
+            response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder);
             return ok(response);
         } catch (WrappedResponse wr) {
             return wr.getResponse();

From 2c2cd86f7a1d0cf994cded37ffbf71e66bb824cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= <pkiraly@gwdg.de>
Date: Tue, 20 Jun 2023 18:29:18 +0200
Subject: [PATCH 036/137] #9650 backwards compatibility, release note

---
 .../9650-5-improve-list-linked-dataverses-API.md             | 5 +++++
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java     | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 doc/release-notes/9650-5-improve-list-linked-dataverses-API.md

diff --git a/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md b/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md
new file mode 100644
index 00000000000..8c79955891b
--- /dev/null
+++ b/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md
@@ -0,0 +1,5 @@
+The following API have been added:
+
+/api/datasets/{datasetId}/links
+
+It lists the linked dataverses to a dataset. It can be executed only by administrators.
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 1700c4fbcf5..24b20b8b882 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2085,7 +2085,7 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id")
             JsonObjectBuilder response = Json.createObjectBuilder();
             response.add("id", datasetId);
             response.add("identifier", dataset.getIdentifier());
-            response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder);
+            response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder);
             return ok(response);
         } catch (WrappedResponse wr) {
             return wr.getResponse();

From 6228d3f030a331487e9237958626f171cb52c623 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 12 Sep 2024 12:59:33 -0400
Subject: [PATCH 037/137] adding tests, doc changes, and formatting fix

---
 .../source/admin/dataverses-datasets.rst      | 12 ++++++--
 .../harvard/iq/dataverse/api/Datasets.java    | 12 ++++----
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 28 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/MoveIT.java  |  7 ++---
 4 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst
index b4d365c4fd4..7c03a6f80c0 100644
--- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst
+++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst
@@ -129,15 +129,21 @@ Lists the link(s) created between a dataset and a Dataverse collection (see the
 
     curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/datasets/$linked-dataset-id/links
 
-It returns a list in the following format:
+It returns a list in the following format (new format as of v6.4):
 
 .. code-block:: json
 
   {
     "status": "OK",
     "data": {
-      "dataverses that link to dataset id 56782": [
-        "crc990 (id 18802)"
+      "id": 5,
+      "identifier": "FK2/OTCWMM",
+      "linked-dataverses": [
+        {
+          "id": 2,
+          "alias": "dataverse1",
+          "displayName": "Lab experiments 2023 June"
+        }
       ]
     }
   }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 24b20b8b882..e0d3bf6126e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2076,16 +2076,16 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id")
             List<Dataverse> dvsThatLinkToThisDatasetId = dataverseSvc.findDataversesThatLinkToThisDatasetId(datasetId);
             JsonArrayBuilder dataversesThatLinkToThisDatasetIdBuilder = Json.createArrayBuilder();
             for (Dataverse dataverse : dvsThatLinkToThisDatasetId) {
-                JsonObjectBuilder entry = Json.createObjectBuilder();
-                entry.add("id", dataverse.getId());
-                entry.add("alias", dataverse.getAlias());
-                entry.add("displayName", dataverse.getDisplayName());
-                dataversesThatLinkToThisDatasetIdBuilder.add(entry);
+                JsonObjectBuilder datasetBuilder = Json.createObjectBuilder();
+                datasetBuilder.add("id", dataverse.getId());
+                datasetBuilder.add("alias", dataverse.getAlias());
+                datasetBuilder.add("displayName", dataverse.getDisplayName());
+                dataversesThatLinkToThisDatasetIdBuilder.add(datasetBuilder.build());
             }
             JsonObjectBuilder response = Json.createObjectBuilder();
             response.add("id", datasetId);
             response.add("identifier", dataset.getIdentifier());
-            response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder);
+            response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder);
             return ok(response);
         } catch (WrappedResponse wr) {
             return wr.getResponse();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index f52aa4fe9bd..b7bdaeebbfc 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -2963,6 +2963,34 @@ public void testLinkingDatasets() {
         linkDataset.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
+        // Link another to test the list of linked datasets
+        Response createDataverse3 = UtilIT.createRandomDataverse(apiToken);
+        createDataverse3.prettyPrint();
+        createDataverse3.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+        String dataverse3Alias = UtilIT.getAliasFromResponse(createDataverse3);
+        Integer dataverse3Id = UtilIT.getDatasetIdFromResponse(createDataverse3);
+        linkDataset = UtilIT.linkDataset(datasetPid, dataverse3Alias, superuserApiToken);
+        linkDataset.prettyPrint();
+        linkDataset.then().assertThat()
+                .statusCode(OK.getStatusCode());
+        // get the list in Json format
+        Response linkDatasetsResponse = UtilIT.getDatasetLinks(datasetPid, superuserApiToken);
+        linkDatasetsResponse.prettyPrint();
+        linkDatasetsResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+        JsonObject linkDatasets = Json.createReader(new StringReader(linkDatasetsResponse.asString())).readObject();
+        JsonArray lst = linkDatasets.getJsonObject("data").getJsonArray("linked-dataverses");
+        List<Integer> ids = List.of(dataverse2Id, dataverse3Id);
+        List<Integer> uniqueids = new ArrayList<>();
+        assertEquals(ids.size(), lst.size());
+        for (int i = 0; i < lst.size(); i++) {
+            int id = lst.getJsonObject(i).getInt("id");
+            assertTrue(ids.contains(id));
+            assertFalse(uniqueids.contains(id));
+            uniqueids.add(id);
+        }
+
 //Experimental code for trying to trick test into thinking the dataset has been harvested
 /*
 createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverse1Alias, apiToken);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java
index f7135ce7f3b..8951b0bd42e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java
@@ -278,8 +278,8 @@ public void testMoveLinkedDataset() {
                 .body("message", equalTo("Use the query parameter forceMove=true to complete the move. This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. "));
 
         JsonObject linksBeforeData = Json.createReader(new StringReader(getLinksBefore.asString())).readObject();
-        assertEquals("OK", linksBeforeData.getString("status"));
-        assertEquals(dataverse2Alias + " (id " + dataverse2Id + ")", linksBeforeData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).getString(0));
+        assertEquals(datasetId, linksBeforeData.getJsonObject("data").getInt("id"));
+        assertEquals(dataverse2Id, linksBeforeData.getJsonObject("data").getJsonArray("linked-dataverses").get(0).asJsonObject().getInt("id"));
 
         boolean forceMove = true;
         Response forceMoveLinkedDataset = UtilIT.moveDataset(datasetId.toString(), dataverse2Alias, forceMove, superuserApiToken);
@@ -308,8 +308,7 @@ public void testMoveLinkedDataset() {
 
         JsonObject linksAfterData = Json.createReader(new StringReader(getLinksAfter.asString())).readObject();
         assertEquals("OK", linksAfterData.getString("status"));
-        assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).size());
-
+        assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("linked-dataverses").size());
     }
     
     @Test

From 54fad55ed773dab96f023ee95dc980294289cec5 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 12 Sep 2024 14:42:30 -0400
Subject: [PATCH 038/137] update doc

---
 doc/sphinx-guides/source/api/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index c5890fd9996..92cd4fc941b 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -12,7 +12,7 @@ v6.4
 
 - **/api/datasets/$dataset-id/modifyRegistration**: Changed from GET to POST
 - **/api/datasets/modifyRegistrationPIDMetadataAll**: Changed from GET to POST
-
+- **/api/datasets/{identifier}/links**: The GET endpoint returns a list of Dataverses linked to the given Dataset. The format of the response has changes for v6.4 making it backward incompatible.
 
 v6.3
 ----

From 7948836af880fcef83d58f659a7c5f8b2577eeec Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 26 Sep 2024 15:56:20 -0400
Subject: [PATCH 039/137] Update the conditions to display image_url

---
 .../iq/dataverse/ThumbnailServiceWrapper.java | 21 +++++++++++--------
 .../harvard/iq/dataverse/api/SearchIT.java    |  8 +++----
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 542cf39cfbe..12805de0348 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -5,6 +5,7 @@
  */
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
@@ -20,7 +21,6 @@
 
 import jakarta.ejb.EJB;
 import jakarta.enterprise.context.RequestScoped;
-import jakarta.inject.Inject;
 import jakarta.inject.Named;
 
 /**
@@ -33,9 +33,8 @@
 public class ThumbnailServiceWrapper implements java.io.Serializable  {
     
     private static final Logger logger = Logger.getLogger(ThumbnailServiceWrapper.class.getCanonicalName());
-    
-    @Inject
-    PermissionsWrapper permissionsWrapper;
+    @EJB
+    PermissionServiceBean permissionService;
     @EJB
     DataverseServiceBean dataverseService;
     @EJB
@@ -49,14 +48,18 @@ public class ThumbnailServiceWrapper implements java.io.Serializable  {
     private Map<Long, DvObject> dvobjectViewMap = new HashMap<>();
     private Map<Long, Boolean> hasThumbMap = new HashMap<>();
 
+    private boolean hasDownloadFilePermission(DvObject dvo) {
+        return permissionService.on(dvo).has(Permission.DownloadFile) ;
+    }
     public String getFileCardImageAsUrl(SolrSearchResult result) {
         DataFile dataFile = result != null && result.getEntity() != null ? ((DataFile) result.getEntity()) : null;
-        if (dataFile == null || result.isHarvested()
+        if (dataFile == null
+                || result.isHarvested()
                 || !isThumbnailAvailable(dataFile)
-                || dataFile.isRestricted()
-                || !dataFile.isReleased()
+                || (dataFile.isRestricted() && !hasDownloadFilePermission(dataFile))
                 || FileUtil.isActivelyEmbargoed(dataFile)
-                || FileUtil.isRetentionExpired(dataFile)) {
+                || FileUtil.isRetentionExpired(dataFile))
+        {
             return null;
         }
         return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + dataFile.getId() + "?imageThumb=true";
@@ -105,7 +108,7 @@ public String getFileCardImageAsBase64Url(SolrSearchResult result) {
             }
 
             if ((!((DataFile)result.getEntity()).isRestricted()
-                        || permissionsWrapper.hasDownloadFilePermission(result.getEntity()))
+                        || hasDownloadFilePermission(result.getEntity()))
                     && isThumbnailAvailable((DataFile) result.getEntity())) {
                 
                 cardImageUrl = ImageThumbConverter.getImageThumbnailAsBase64(
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
index 3a2b684c421..b03c23cd1e2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
@@ -1308,8 +1308,8 @@ public void testSearchFilesAndUrlImages() {
                 .statusCode(200);
         pathToFile = "src/main/webapp/resources/js/mydata.js";
         Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
-        uploadImage.prettyPrint();
-        uploadImage.then().assertThat()
+        uploadFile.prettyPrint();
+        uploadFile.then().assertThat()
                 .statusCode(200);
 
         Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken);
@@ -1337,7 +1337,7 @@ public void testSearchFilesAndUrlImages() {
                 .statusCode(OK.getStatusCode())
                 .body("data.items[0].type", CoreMatchers.is("dataverse"))
                 .body("data.items[0].url", CoreMatchers.containsString("/dataverse/"))
-                .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("url_image")));
+                .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url")));
 
         searchResp = UtilIT.search("mydata", apiToken);
         searchResp.prettyPrint();
@@ -1345,6 +1345,6 @@ public void testSearchFilesAndUrlImages() {
                 .statusCode(OK.getStatusCode())
                 .body("data.items[0].type", CoreMatchers.is("file"))
                 .body("data.items[0].url", CoreMatchers.containsString("/datafile/"))
-                .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("url_image")));
+                .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url")));
     }
 }

From d52aa58b287994602f7342c92118dad6a085fe55 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Thu, 26 Sep 2024 16:01:57 -0400
Subject: [PATCH 040/137] Update the conditions to display image_url

---
 .../java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 12805de0348..46736da73d4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -58,8 +58,7 @@ public String getFileCardImageAsUrl(SolrSearchResult result) {
                 || !isThumbnailAvailable(dataFile)
                 || (dataFile.isRestricted() && !hasDownloadFilePermission(dataFile))
                 || FileUtil.isActivelyEmbargoed(dataFile)
-                || FileUtil.isRetentionExpired(dataFile))
-        {
+                || FileUtil.isRetentionExpired(dataFile)) {
             return null;
         }
         return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + dataFile.getId() + "?imageThumb=true";

From 01f0bd3ed8428b946a394dcbcf94c61207341d5d Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 27 Sep 2024 16:14:25 -0400
Subject: [PATCH 041/137] abstract cleaning to separate method

---
 .../pidproviders/doi/AbstractDOIProvider.java | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
index fae40ddb77e..70ce1ec4c14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
@@ -28,27 +28,23 @@ public abstract class AbstractDOIProvider extends AbstractPidProvider {
     public AbstractDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList) {
         super(id, label, DOI_PROTOCOL, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList);
         //Create case insensitive (converted toUpperCase) managedSet and excludedSet
-        HashSet<String> cleanManagedSet = new HashSet<String>();
-        for(String entry: managedSet) {
-            if(entry.startsWith(DOI_PROTOCOL)) {
-                cleanManagedSet.add(DOI_PROTOCOL + entry.substring(DOI_PROTOCOL.length()).toUpperCase());
-            } else {
-                logger.warning("Non-DOI found in managedSet of pidProvider id: " + getId() + ": " + entry + ". Entry is being dropped.");
-            }
-        }
-        managedSet = cleanManagedSet;
-        HashSet<String> cleanExcludedSet = new HashSet<String>();
-        for(String entry: excludedSet) {
+        managedSet = clean(managedSet, "managed");
+        excludedSet = clean(excludedSet, "excluded");
+    }
+
+    private HashSet<String> clean(HashSet<String> originalSet, String setName) {
+        HashSet<String> cleanSet = new HashSet<String>();
+        for(String entry: originalSet) {
             if(entry.startsWith(DOI_PROTOCOL)) {
-                cleanExcludedSet.add(DOI_PROTOCOL + entry.substring(DOI_PROTOCOL.length()).toUpperCase());
+                cleanSet.add(DOI_PROTOCOL + entry.substring(DOI_PROTOCOL.length()).toUpperCase());
             } else {
-                logger.warning("Non-DOI found in excludedSet of pidProvider id: " + getId() + ": " + entry + ". Entry is being dropped.");
+                logger.warning("Non-DOI found in " + setName + " set of pidProvider id: " + getId() + ": " + entry + ". Entry is being dropped.");
             }
         }
-        excludedSet = cleanExcludedSet;
+        return cleanSet;
     }
 
-    //For Unmanged provider
+    //For Unmanaged provider
     public AbstractDOIProvider(String name, String label) {
         super(name, label, DOI_PROTOCOL);
     }

From 700c39fe8cfb93e69c20ce3ed44f617b7d2ba8b3 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 30 Sep 2024 13:34:08 -0400
Subject: [PATCH 042/137] add release note

---
 .../10886-update-to-conditions-to-display-image_url.md      | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/release-notes/10886-update-to-conditions-to-display-image_url.md

diff --git a/doc/release-notes/10886-update-to-conditions-to-display-image_url.md b/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
new file mode 100644
index 00000000000..03bd8299d45
--- /dev/null
+++ b/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
@@ -0,0 +1,6 @@
+Search API (/api/search) responses for Datafiles include image_url for the thumbnail if each of the following are true:
+1. The DataFile is not Harvested
+2. A Thumbnail is available for the Datafile
+3. If the Datafile is Restricted then the caller must have Download File Permission for the Datafile
+4. The Datafile is NOT actively embargoed
+5. The Datafile's retention is NOT expired

From 48dead84d3ce2d7b093eb1b1c6e39e475cd9aa9a Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 30 Sep 2024 13:35:11 -0400
Subject: [PATCH 043/137] add release note

---
 .../10886-update-to-conditions-to-display-image_url.md          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/10886-update-to-conditions-to-display-image_url.md b/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
index 03bd8299d45..a7adda11840 100644
--- a/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
+++ b/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
@@ -3,4 +3,4 @@ Search API (/api/search) responses for Datafiles include image_url for the thumb
 2. A Thumbnail is available for the Datafile
 3. If the Datafile is Restricted then the caller must have Download File Permission for the Datafile
 4. The Datafile is NOT actively embargoed
-5. The Datafile's retention is NOT expired
+5. The Datafile's retention has NOT expired

From 7d1e6feefe18611cc34b90784da5c79a82df8cc0 Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Tue, 1 Oct 2024 12:00:51 +0200
Subject: [PATCH 044/137] replace ZipInputStream with ZipFile

---
 .../impl/CreateNewDataFilesCommand.java       | 331 ++++++++----------
 .../ingest/IngestServiceShapefileHelper.java  |  67 ++--
 .../harvard/iq/dataverse/util/FileUtil.java   |  13 +-
 .../iq/dataverse/util/ShapefileHandler.java   | 149 +++-----
 .../command/impl/CreateNewDataFilesTest.java  | 187 ++++++++++
 .../util/shapefile/ShapefileHandlerTest.java  |  38 +-
 .../own-cloud-downloads/greetings.zip         | Bin 0 -> 679 bytes
 .../resources/own-cloud-downloads/shapes.zip  | Bin 0 -> 4336 bytes
 8 files changed, 429 insertions(+), 356 deletions(-)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
 create mode 100644 src/test/resources/own-cloud-downloads/greetings.zip
 create mode 100644 src/test/resources/own-cloud-downloads/shapes.zip

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 3a21345448b..e543606e039 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -2,34 +2,29 @@
 
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
 import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
-import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
-//import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
-import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
-import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
-import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
-import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport;
-import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType;
-import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension;
-import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory;
-import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile;
-import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType;
 import edu.harvard.iq.dataverse.util.ShapefileHandler;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.file.BagItFileHandler;
 import edu.harvard.iq.dataverse.util.file.BagItFileHandlerFactory;
 import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;
+import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
+import jakarta.enterprise.inject.spi.CDI;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -42,7 +37,7 @@
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Enumeration;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -51,12 +46,17 @@
 import java.util.Set;
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipFile;
 import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import jakarta.enterprise.inject.spi.CDI;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
+import java.util.zip.ZipFile;
+
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
+import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport;
+import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType;
+import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension;
+import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory;
+import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile;
+import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType;
 
 /**
  *
@@ -140,9 +140,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
 
 
         if (newStorageIdentifier == null) {
-            if (getFilesTempDirectory() != null) {
+            var filesTempDirectory = getFilesTempDirectory();
+            if (filesTempDirectory != null) {
                 try {
-                    tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
+                    tempFile = Files.createTempFile(Paths.get(filesTempDirectory), "tmp", "upload");
                     // "temporary" location is the key here; this is why we are not using
                     // the DataStore framework for this - the assumption is that
                     // temp files will always be stored on the local filesystem.
@@ -260,10 +261,6 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 // DataFile objects from its contents:
             } else if (finalType.equals("application/zip")) {
 
-                ZipFile zipFile = null;
-                ZipInputStream unZippedIn = null;
-                ZipEntry zipEntry = null;
-
                 int fileNumberLimit = ctxt.systemConfig().getZipUploadFilesLimit();
                 Long combinedUnzippedFileSize = 0L;
 
@@ -271,14 +268,14 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     Charset charset = null;
                     /*
                 	TODO: (?)
-                	We may want to investigate somehow letting the user specify 
+                	We may want to investigate somehow letting the user specify
                 	the charset for the filenames in the zip file...
-                    - otherwise, ZipInputStream bails out if it encounteres a file 
-                	name that's not valid in the current charest (i.e., UTF-8, in 
-                    our case). It would be a bit trickier than what we're doing for 
-                    SPSS tabular ingests - with the lang. encoding pulldown menu - 
+                    - otherwise, ZipInputStream bails out if it encounteres a file
+                	name that's not valid in the current charest (i.e., UTF-8, in
+                    our case). It would be a bit trickier than what we're doing for
+                    SPSS tabular ingests - with the lang. encoding pulldown menu -
                 	because this encoding needs to be specified *before* we upload and
-                    attempt to unzip the file. 
+                    attempt to unzip the file.
                 	        -- L.A. 4.0 beta12
                 	logger.info("default charset is "+Charset.defaultCharset().name());
                 	if (Charset.isSupported("US-ASCII")) {
@@ -287,25 +284,21 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     	if (charset != null) {
                        	    logger.info("was able to obtain charset for US-ASCII");
                     	}
-                    
+
                 	 }
                      */
 
-                    /** 
-                     * Perform a quick check for how many individual files are 
-                     * inside this zip archive. If it's above the limit, we can 
-                     * give up right away, without doing any unpacking. 
+                    /**
+                     * Perform a quick check for how many individual files are
+                     * inside this zip archive. If it's above the limit, we can
+                     * give up right away, without doing any unpacking.
                      * This should be a fairly inexpensive operation, we just need
-                     * to read the directory at the end of the file. 
+                     * to read the directory at the end of the file.
                      */
-                    
-                    if (charset != null) {
-                        zipFile = new ZipFile(tempFile.toFile(), charset);
-                    } else {
-                        zipFile = new ZipFile(tempFile.toFile());
-                    }
+
+
                     /**
-                     * The ZipFile constructors above will throw ZipException - 
+                     * The ZipFile constructors in openZipFile will throw ZipException -
                      * a type of IOException - if there's something wrong 
                      * with this file as a zip. There's no need to intercept it
                      * here, it will be caught further below, with other IOExceptions,
@@ -313,8 +306,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                      * then attempt to save it as is.
                      */
 
-                    int numberOfUnpackableFiles = 0; 
-                     
+                    int numberOfUnpackableFiles = 0;
+
                     /**
                      * Note that we can't just use zipFile.size(),
                      * unfortunately, since that's the total number of entries,
@@ -323,83 +316,48 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                      * that are files.
                      */
 
-                    for (Enumeration<? extends ZipEntry> entries = zipFile.entries(); entries.hasMoreElements();) {
-                        ZipEntry entry = entries.nextElement();
-                        logger.fine("inside first zip pass; this entry: "+entry.getName());
-                        if (!entry.isDirectory()) {
-                            String shortName = entry.getName().replaceFirst("^.*[\\/]", "");
-                            // ... and, finally, check if it's a "fake" file - a zip archive entry
-                            // created for a MacOS X filesystem element: (these
-                            // start with "._") 
-                            if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
-                                numberOfUnpackableFiles++;
-                                if (numberOfUnpackableFiles > fileNumberLimit) {
-                                    logger.warning("Zip upload - too many files in the zip to process individually.");
-                                    warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
-                                            + "); please upload a zip archive with fewer files, if you want them to be ingested "
-                                            + "as individual DataFiles.";
-                                    throw new IOException();
-                                }
-                                // In addition to counting the files, we can
-                                // also check the file size while we're here, 
-                                // provided the size limit is defined; if a single 
-                                // file is above the individual size limit, unzipped,
-                                // we give up on unpacking this zip archive as well: 
-                                if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) {
-                                    throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit)));
-                                }
-                                // Similarly, we want to check if saving all these unpacked 
-                                // files is going to push the disk usage over the 
-                                // quota:
-                                if (storageQuotaLimit != null) {
-                                    combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize();
-                                    if (combinedUnzippedFileSize > storageQuotaLimit) {
-                                        //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
-                                        // change of plans: if the unzipped content inside exceeds the remaining quota, 
-                                        // we reject the upload outright, rather than accepting the zip 
-                                        // file as is. 
-                                        throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.unzipped.quota_exceeded"), bytesToHumanReadable(storageQuotaLimit)), this);
-                                    }
+                    try (var zipFile = openZipFile(tempFile, charset)) {
+                        for (var entry : filteredZipEntries(zipFile)) {
+                            logger.fine("inside first zip pass; this entry: " + entry.getName());
+                            numberOfUnpackableFiles++;
+                            if (numberOfUnpackableFiles > fileNumberLimit) {
+                                logger.warning("Zip upload - too many files in the zip to process individually.");
+                                warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
+                                                 + "); please upload a zip archive with fewer files, if you want them to be ingested "
+                                                 + "as individual DataFiles.";
+                                throw new IOException();
+                            }
+                            // In addition to counting the files, we can
+                            // also check the file size while we're here,
+                            // provided the size limit is defined; if a single
+                            // file is above the individual size limit, unzipped,
+                            // we give up on unpacking this zip archive as well:
+                            if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) {
+                                throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit)));
+                            }
+                            // Similarly, we want to check if saving all these unpacked
+                            // files is going to push the disk usage over the
+                            // quota:
+                            if (storageQuotaLimit != null) {
+                                combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize();
+                                if (combinedUnzippedFileSize > storageQuotaLimit) {
+                                    //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                                    // change of plans: if the unzipped content inside exceeds the remaining quota,
+                                    // we reject the upload outright, rather than accepting the zip
+                                    // file as is.
+                                    throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.unzipped.quota_exceeded"), bytesToHumanReadable(storageQuotaLimit)), this);
                                 }
                             }
                         }
                     }
-                    
+
                     // OK we're still here - that means we can proceed unzipping. 
                     
-                    // Close the ZipFile, re-open as ZipInputStream: 
-                    zipFile.close(); 
                     // reset:
                     combinedUnzippedFileSize = 0L;
 
-                    if (charset != null) {
-                        unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset);
-                    } else {
-                        unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));
-                    }
-
-                    while (true) {
-                        try {
-                            zipEntry = unZippedIn.getNextEntry();
-                        } catch (IllegalArgumentException iaex) {
-                            // Note:
-                            // ZipInputStream documentation doesn't even mention that
-                            // getNextEntry() throws an IllegalArgumentException!
-                            // but that's what happens if the file name of the next
-                            // entry is not valid in the current CharSet.
-                            // -- L.A.
-                            warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is.";
-                            logger.warning(warningMessage);
-                            throw new IOException();
-                        }
-
-                        if (zipEntry == null) {
-                            break;
-                        }
-                        // Note that some zip entries may be directories - we
-                        // simply skip them:
-
-                        if (!zipEntry.isDirectory()) {
+                    try (var zipFile = openZipFile(tempFile, charset)) {
+                        for (var entry : filteredZipEntries(zipFile)) {
                             if (datafiles.size() > fileNumberLimit) {
                                 logger.warning("Zip upload - too many files.");
                                 warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
@@ -407,72 +365,55 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                                         + "as individual DataFiles.";
                                 throw new IOException();
                             }
-
-                            String fileEntryName = zipEntry.getName();
+                            var fileEntryName = entry.getName();
+                            var shortName = getShortName(fileEntryName);
                             logger.fine("ZipEntry, file: " + fileEntryName);
+                            String storageIdentifier = FileUtil.generateStorageIdentifier();
+                            File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier);
+                            Files.copy(zipFile.getInputStream(entry), unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                            // No need to check the size of this unpacked file against the size limit,
+                            // since we've already checked for that in the first pass.
+                            DataFile datafile = FileUtil.createSingleDataFile(version, null, storageIdentifier, shortName,
+                                MIME_TYPE_UNDETERMINED_DEFAULT,
+                                ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false);
+
+                            if (!fileEntryName.equals(shortName)) {
+                                // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
+                                // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all
+                                // the leading, trailing and duplicate slashes; then replace all the characters that
+                                // don't pass our validation rules.
+                                String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", "");
+                                directoryName = StringUtil.sanitizeFileDirectory(directoryName, true);
+                                // if (!"".equals(directoryName)) {
+                                if (!StringUtil.isEmpty(directoryName)) {
+                                    logger.fine("setting the directory label to " + directoryName);
+                                    datafile.getFileMetadata().setDirectoryLabel(directoryName);
+                                }
+                            }
 
-                            if (fileEntryName != null && !fileEntryName.equals("")) {
-
-                                String shortName = fileEntryName.replaceFirst("^.*[\\/]", "");
-
-                                // Check if it's a "fake" file - a zip archive entry
-                                // created for a MacOS X filesystem element: (these
-                                // start with "._")
-                                if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
-                                    // OK, this seems like an OK file entry - we'll try
-                                    // to read it and create a DataFile with it:
-
-                                    String storageIdentifier = FileUtil.generateStorageIdentifier();
-                                    File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier);
-                                    Files.copy(unZippedIn, unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
-                                    // No need to check the size of this unpacked file against the size limit, 
-                                    // since we've already checked for that in the first pass.
-                                    
-                                    DataFile datafile = FileUtil.createSingleDataFile(version, null, storageIdentifier, shortName,
-                                            MIME_TYPE_UNDETERMINED_DEFAULT,
-                                            ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false);
-                                    
-                                    if (!fileEntryName.equals(shortName)) {
-                                        // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
-                                        // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all 
-                                        // the leading, trailing and duplicate slashes; then replace all the characters that 
-                                        // don't pass our validation rules.
-                                        String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", "");
-                                        directoryName = StringUtil.sanitizeFileDirectory(directoryName, true);
-                                        // if (!"".equals(directoryName)) {
-                                        if (!StringUtil.isEmpty(directoryName)) {
-                                            logger.fine("setting the directory label to " + directoryName);
-                                            datafile.getFileMetadata().setDirectoryLabel(directoryName);
-                                        }
-                                    }
+                            if (datafile != null) {
+                                // We have created this datafile with the mime type "unknown";
+                                // Now that we have it saved in a temporary location,
+                                // let's try and determine its real type:
 
-                                    if (datafile != null) {
-                                        // We have created this datafile with the mime type "unknown";
-                                        // Now that we have it saved in a temporary location,
-                                        // let's try and determine its real type:
-
-                                        String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
-
-                                        try {
-                                            recognizedType = determineFileType(unzippedFile, shortName);
-                                            // null the File explicitly, to release any open FDs:
-                                            unzippedFile = null;
-                                            logger.fine("File utility recognized unzipped file as " + recognizedType);
-                                            if (recognizedType != null && !recognizedType.equals("")) {
-                                                datafile.setContentType(recognizedType);
-                                            }
-                                        } catch (Exception ex) {
-                                            logger.warning("Failed to run the file utility mime type check on file " + fileName);
-                                        }
-
-                                        datafiles.add(datafile);
-                                        combinedUnzippedFileSize += datafile.getFilesize();
+                                String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
+
+                                try {
+                                    recognizedType = determineFileType(unzippedFile, shortName);
+                                    // null the File explicitly, to release any open FDs:
+                                    unzippedFile = null;
+                                    logger.fine("File utility recognized unzipped file as " + recognizedType);
+                                    if (recognizedType != null && !recognizedType.equals("")) {
+                                        datafile.setContentType(recognizedType);
                                     }
+                                } catch (Exception ex) {
+                                    logger.warning("Failed to run the file utility mime type check on file " + fileName);
                                 }
+
+                                datafiles.add(datafile);
+                                combinedUnzippedFileSize += datafile.getFilesize();
                             }
                         }
-                        unZippedIn.closeEntry();
-
                     }
 
                 } catch (IOException ioex) {
@@ -494,18 +435,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     //warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));
                     //datafiles.clear();
                     throw new CommandExecutionException(fesqx.getMessage(), fesqx, this);
-                }*/ finally {
-                    if (zipFile != null) {
-                        try {
-                            zipFile.close();
-                        } catch (Exception zEx) {}
-                    }
-                    if (unZippedIn != null) {
-                        try {
-                            unZippedIn.close();
-                        } catch (Exception zEx) {}
-                    }
-                }
+                }*/
                 if (!datafiles.isEmpty()) {
                     // remove the uploaded zip file:
                     try {
@@ -591,7 +521,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 // The try-catch is due to error encountered in using NFS for stocking file,
                 // cf. https://github.com/IQSS/dataverse/issues/5909
                 try {
-                    FileUtils.deleteDirectory(rezipFolder);
+                    if (rezipFolder!=null)
+                        FileUtils.deleteDirectory(rezipFolder);
                 } catch (IOException ioex) {
                     // do nothing - it's a temp folder.
                     logger.warning("Could not remove temp folder, error message : " + ioex.getMessage());
@@ -730,7 +661,37 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
 
         return CreateDataFileResult.error(fileName, finalType);
     }   // end createDataFiles
-    
+
+    private static List<? extends ZipEntry> filteredZipEntries(ZipFile zipFile) {
+        var entries = Collections.list(zipFile.entries()).stream().filter(e -> {
+            var entryName = e.getName();
+            logger.fine("ZipEntry, file: " + entryName);
+            return !e.isDirectory() && !entryName.isEmpty() && !isFileToSkip(entryName);
+        }).toList();
+        return entries;
+    }
+
+    private static ZipFile openZipFile(Path tempFile, Charset charset) throws IOException {
+        if (charset != null) {
+            return new ZipFile(tempFile.toFile(), charset);
+        }
+        else {
+            return new ZipFile(tempFile.toFile());
+        }
+    }
+
+    private static boolean isFileToSkip(String fileName) {
+        // check if it's a "fake" file - a zip archive entry
+        // created for a MacOS X filesystem element: (these
+        // start with "._")
+        var shortName = getShortName(fileName);
+        return shortName.startsWith("._") || shortName.startsWith(".DS_Store") || "".equals(shortName);
+    }
+
+    private static String getShortName(String fileName) {
+        return fileName.replaceFirst("^.*[\\/]", "");
+    }
+
     @Override
     public Map<String, Set<Permission>> getRequiredPermissions() {
         Map<String, Set<Permission>> ret = new HashMap<>();
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java
index 8c5dad237b1..27a2ab99376 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java
@@ -100,71 +100,48 @@ public IngestServiceShapefileHelper(File zippedShapefile, File rezipFolder){
         //this.processFile(zippedShapefile, rezipFolder);
 
     }
-    
-   private FileInputStream getFileInputStream(File fileObject){
-       if (fileObject==null){
-           return null;
-       }
-       try {
+
+    private FileInputStream getFileInputStream(File fileObject){
+        if (fileObject==null){
+            return null;
+        }
+        try {
             return new FileInputStream(fileObject);
         } catch (FileNotFoundException ex) {
             logger.severe("Failed to create FileInputStream from File: " + fileObject.getAbsolutePath());
             return null;
         }
-   }
-   
-   private void closeFileInputStream(FileInputStream fis){
-       if (fis==null){
-           return;
-       }
+    }
+
+    private void closeFileInputStream(FileInputStream fis){
+        if (fis==null){
+            return;
+        }
         try {
-            fis.close();            
+            fis.close();
         } catch (IOException ex) {
             logger.info("Failed to close FileInputStream");
         }
-   }
-   
+    }
+
     public boolean processFile() {
        
        if ((!isValidFile(this.zippedShapefile))||(!isValidFolder(this.rezipFolder))){
             return false;
         }
-        
-       // (1) Use the ShapefileHandler to the .zip for a shapefile
-       //
-        FileInputStream shpfileInputStream = this.getFileInputStream(zippedShapefile);
-        if (shpfileInputStream==null){
-            return false;
-        }
-        
-        this.shpHandler = new ShapefileHandler(shpfileInputStream);
-        if (!shpHandler.containsShapefile()){
-            logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
-            return false;
-        }
-
-        this.closeFileInputStream(shpfileInputStream);
-        
-        //  (2) Rezip the shapefile pieces
-        logger.info("rezipFolder: " + rezipFolder.getAbsolutePath());
-        shpfileInputStream = this.getFileInputStream(zippedShapefile);
-        if (shpfileInputStream==null){
-            return false;
-        }
-
-        boolean rezipSuccess;
         try {
-            rezipSuccess = shpHandler.rezipShapefileSets(shpfileInputStream, rezipFolder);
+            this.shpHandler = new ShapefileHandler(zippedShapefile);
+            if (!shpHandler.containsShapefile()){
+                logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
+                return false;
+            }
+            logger.info("rezipFolder: " + rezipFolder.getAbsolutePath());
+            return shpHandler.rezipShapefileSets(rezipFolder);
         } catch (IOException ex) {
             logger.severe("Shapefile was not correctly unpacked/repacked");
             logger.severe("shpHandler message: " + shpHandler.errorMessage);
             return false;
         }
-        
-        this.closeFileInputStream(shpfileInputStream);
-        
-        return rezipSuccess;
-     
         //   return createDataFiles(rezipFolder);
         
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index a0c32d5c8ce..991682ec8e8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -525,15 +525,18 @@ public static String determineFileType(File f, String fileName) throws IOExcepti
             // Check for shapefile extensions as described here: http://en.wikipedia.org/wiki/Shapefile
             //logger.info("Checking for shapefile");
 
-            ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(f));
+            ShapefileHandler shp_handler = new ShapefileHandler(f);
              if (shp_handler.containsShapefile()){
               //  logger.info("------- shapefile FOUND ----------");
                  fileType = ShapefileHandler.SHAPEFILE_FILE_TYPE; //"application/zipped-shapefile";
              }
-
-            Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
-             if(bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
-                 fileType = BagItFileHandler.FILE_TYPE;
+             try {
+                 Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
+                 if (bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
+                     fileType = BagItFileHandler.FILE_TYPE;
+                 }
+             } catch (Exception e) {
+                 logger.warning("Error checking for BagIt package: " + e.getMessage());
              }
         } 
         
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
index f1440cc3c02..20dc6d9c26a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
@@ -1,23 +1,21 @@
 package edu.harvard.iq.dataverse.util;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.Date;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
 import java.util.HashMap;
 import java.util.*;
 
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
-import java.util.logging.Level;
+
 import java.util.logging.Logger;
 import org.apache.commons.io.FileUtils;
 
@@ -43,11 +41,10 @@
  *                                  "shape1.pdf", "README.md", "shape_notes.txt"
  * 
  *  Code Example:  
- *          FileInputStream shp_file_input_stream = new FileInputStream(new File("zipped_shapefile.zip"))
- *          ShapefileHandler shp_handler = new ShapefileHandler(shp_file_input_stream);
+ *          ShapefileHandler shp_handler = new ShapefileHandler(new File("zipped_shapefile.zip"));
  *          if (shp_handler.containsShapefile()){
  *              File rezip_folder = new File("~/folder_for_rezipping");
- *              boolean rezip_success = shp_handler.rezipShapefileSets(shp_file_input_stream, rezip_folder );
+ *              boolean rezip_success = shp_handler.rezipShapefileSets(rezip_folder );
  *              if (!rezip_success){
  *                  // rezip failed, should be an error message (String) available
                     System.out.println(shp_handler.error_message);
@@ -74,7 +71,8 @@ public class ShapefileHandler{
     public final static String SHP_XML_EXTENSION = "shp.xml";
     public final static String BLANK_EXTENSION = "__PLACEHOLDER-FOR-BLANK-EXTENSION__";
     public final static List<String> SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", "qpj", "qmd", SHP_XML_EXTENSION);
-    
+    private final File zipFile;
+
     public boolean DEBUG = false;
         
     private boolean zipFileProcessed = false;
@@ -97,9 +95,6 @@ public class ShapefileHandler{
     private Map<String, List<String>> fileGroups = new HashMap<>();
     
     private List<File> finalRezippedFiles = new ArrayList<>();
-    
-    private String outputFolder = "unzipped";
-    private String rezippedFolder = "rezipped";
 
     // Debug helper
     private void msg(String s){
@@ -116,40 +111,28 @@ private void msgt(String s){
     }
 
     /*
-        Constructor, start with filename
-    */
-    public ShapefileHandler(String filename){
-
-        if (filename==null){
-            this.addErrorMessage("The filename was null");
-            return;
-        }
-        
-        FileInputStream zip_file_stream;
-        try {
-            zip_file_stream = new FileInputStream(new File(filename));
-        } catch (FileNotFoundException ex) {
-            this.addErrorMessage("The file was not found");
+         Constructor, start with File
+     */
+    public ShapefileHandler(File zip_file) throws IOException {
+        zipFile = zip_file;
+        if (zip_file == null) {
+            this.addErrorMessage("The file was null");
             return;
         }
-        
-       this.examineZipfile(zip_file_stream);
 
-    }
-
-   
-   /*
-        Constructor, start with FileInputStream
-    */
-   public ShapefileHandler(FileInputStream zip_file_stream){
-
-        if (zip_file_stream==null){
-            this.addErrorMessage("The zip_file_stream was null");
-            return;
+        try (var zip_file_object = new ZipFile(zip_file)) {
+            this.examineZipfile(zip_file_object);
+        }
+        catch (FileNotFoundException ex) {
+            // While this constructor had a FileInputStream as argument:
+            //  FileUtil.determineFileType threw this exception before calling the constructor with a FileInputStream
+            //  IngestServiceShapefileHelper.processFile won´t call this constructor if the file is not valid hence does not exist.
+            //  When the file would have disappeared in the meantime, it would have produced a slightly different error message.
+            logger.severe("File not found: " + zip_file.getAbsolutePath());
+            throw ex;
         }
-        this.examineZipfile(zip_file_stream);
     }
-    
+
     public List<File> getFinalRezippedFiles(){
         return this.finalRezippedFiles;
     }
@@ -291,26 +274,19 @@ inside the uploaded zip file (issue #6873). To achieve this, we recreate
         subfolders in the FileMetadata of the newly created DataFiles. 
         (-- L.A. 09/2020) 
     */
-    private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File target_directory){
+    private boolean unzipFilesToDirectory(ZipFile zipfileInput, File target_directory){
         logger.fine("unzipFilesToDirectory: " + target_directory.getAbsolutePath() );
 
-        if (zipfile_input_stream== null){
-            this.addErrorMessage("unzipFilesToDirectory. The zipfile_input_stream is null.");
-            return false;
-        }
         if (!target_directory.isDirectory()){
              this.addErrorMessage("This directory does not exist: " + target_directory.getAbsolutePath());
             return false;
         }
 
-       List<String> unzippedFileNames = new ArrayList<>(); 
-       
-       ZipInputStream zipStream = new ZipInputStream(zipfile_input_stream);
+       List<String> unzippedFileNames = new ArrayList<>();
+
 
-       ZipEntry origEntry;
-        byte[] buffer = new byte[2048];
         try {
-            while((origEntry = zipStream.getNextEntry())!=null){
+            for(var origEntry : Collections.list(zipfileInput.entries())){
                 
                 String zentryFileName = origEntry.getName();
                 logger.fine("\nOriginal entry name: " + origEntry);
@@ -360,15 +336,10 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File
                     unzippedFileNames.add(outpath);
                 }
                 logger.fine("Write zip file: " + outpath);
-                FileOutputStream fileOutputStream;
-                long fsize = 0;
-                fileOutputStream = new FileOutputStream(outpath);
-                int len;// = 0;
-                while ((len = zipStream.read(buffer)) > 0){
-                    fileOutputStream.write(buffer, 0, len);
-                    fsize+=len;
-                } // end while
-                fileOutputStream.close();
+                try(var inputStream = zipfileInput.getInputStream(origEntry)) {
+                    Files.createDirectories(new File(outpath).getParentFile().toPath());
+                    Files.copy(inputStream, Path.of(outpath), StandardCopyOption.REPLACE_EXISTING);
+                }
             } // end outer while
         } catch (IOException ex) {
             for (StackTraceElement el : ex.getStackTrace()){
@@ -377,19 +348,13 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File
             this.addErrorMessage("Failed to open ZipInputStream entry" +  ex.getMessage());
             return false;
         }
-
-        try {
-             zipStream.close();
-        } catch (IOException ex) {
-             Logger.getLogger(ShapefileHandler.class.getName()).log(Level.SEVERE, null, ex);
-        }
-    return true;
+        return true;
     }
     /*
         Rezip the shapefile(s) into a given directory
         Assumes that the zipfile_input_stream has already been checked!
     */
-    public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rezippedFolder) throws IOException{
+    public boolean rezipShapefileSets(File rezippedFolder) throws IOException{
         logger.fine("rezipShapefileSets");
         //msgt("rezipShapefileSets");
         if (!this.zipFileProcessed){
@@ -400,10 +365,6 @@ public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rez
              this.addErrorMessage("There are no shapefiles here!");
             return false;
         }
-        if (zipfile_input_stream== null){
-            this.addErrorMessage("The zipfile_input_stream is null.");
-            return false;
-        }
         if (rezippedFolder == null){
             this.addErrorMessage("The rezippedFolder is null.");
             return false;
@@ -433,9 +394,11 @@ public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rez
         
         
         // Unzip files!
-        if (!this.unzipFilesToDirectory(zipfile_input_stream, dir_for_unzipping)){
-            this.addErrorMessage("Failed to unzip files.");
-            return false;
+        try(var zipfileObject = new ZipFile(zipFile)) {
+            if (!this.unzipFilesToDirectory(zipfileObject, dir_for_unzipping)) {
+                this.addErrorMessage("Failed to unzip files.");
+                return false;
+            }
         }
         // Redistribute files!
         String target_dirname = rezippedFolder.getAbsolutePath();
@@ -681,27 +644,19 @@ private boolean isFileToSkip(String fname){
     /**************************************
      * Iterate through the zip file contents.
      * Does it contain any shapefiles?
-     *
-     * @param FileInputStream zip_file_stream
      */
-    private boolean examineZipfile(FileInputStream zip_file_stream){
+    private boolean examineZipfile(ZipFile zip_file){
        // msgt("examineZipfile");
-        
-        if (zip_file_stream==null){
-               this.addErrorMessage("The zip file stream was null");
-               return false;
-           }
-        
+
        // Clear out file lists
        this.filesListInDir.clear();
        this.filesizeHash.clear();
        this.fileGroups.clear();
 
-        try{
-            ZipInputStream zipStream = new ZipInputStream(zip_file_stream);
-            ZipEntry entry;
-            List<String> hiddenDirectories = new ArrayList<>();
-            while((entry = zipStream.getNextEntry())!=null){
+       try{
+           List<String> hiddenDirectories = new ArrayList<>();
+           for(var entry : Collections.list(zip_file.entries())){
+
                 String zentryFileName = entry.getName();
                 boolean isDirectory = entry.isDirectory();
 
@@ -748,8 +703,6 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
                     this.filesizeHash.put(unzipFilePath, entry.getSize());
                 }
            } // end while
-           
-           zipStream.close();
 
            if (this.filesListInDir.isEmpty()){
                errorMessage = "No files in zipStream";
@@ -759,13 +712,8 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
            this.zipFileProcessed = true;
            return true;
 
-       }catch(ZipException ex){
-               this.addErrorMessage("ZipException");
-               msgt("ZipException");
-               return false;
-
        }catch(IOException ex){
-           //ex.printStackTrace(); 
+           //ex.printStackTrace();
            this.addErrorMessage("IOException File name");
            msgt("IOException");
            return false;
@@ -773,9 +721,6 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
            this.addErrorMessage("IllegalArgumentException when parsing zipfile");
            msgt("IllegalArgumentException when parsing zipfile");
            return false;
-           
-       }finally{
-           
        }
 
    } // end examineFile
diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
new file mode 100644
index 00000000000..1262984eb27
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
@@ -0,0 +1,187 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
+import edu.harvard.iq.dataverse.util.JhoveFileType;
+import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
+import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+
+import static edu.harvard.iq.dataverse.DataFile.ChecksumType.MD5;
+import static org.apache.commons.io.file.FilesUncheck.createDirectories;
+import static org.apache.commons.io.file.PathUtils.deleteDirectory;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+
+
+@LocalJvmSettings
+public class CreateNewDataFilesTest {
+    // TODO keep constants for annotations in sync with class name
+    Path testDir = Path.of("target/test/").resolve(getClass().getSimpleName());
+    PrintStream original_stderr;
+
+    @BeforeEach
+    public void cleanTmpDir() throws IOException {
+        original_stderr = System.err;
+        if(testDir.toFile().exists())
+            deleteDirectory(testDir);
+    }
+
+    @AfterEach void restoreStderr() {
+        System.setErr(original_stderr);
+    }
+
+    @Test
+    @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
+    public void execute_fails_to_upload_when_tmp_does_not_exist() throws FileNotFoundException {
+
+        mockTmpLookup();
+        var cmd = createCmd("scripts/search/data/shape/shapefile.zip", mockDatasetVersion(), 1000L, 500L);
+        var ctxt = mockCommandContext(mockSysConfig(true, 0L, MD5, 10));
+
+        assertThatThrownBy(() -> cmd.execute(ctxt))
+            .isInstanceOf(CommandException.class)
+            .hasMessageContaining("Failed to save the upload as a temp file (temp disk space?)")
+            .hasRootCauseInstanceOf(NoSuchFileException.class)
+            .getRootCause()
+            .hasMessageStartingWith("target/test/CreateNewDataFilesTest/tmp/temp/tmp");
+    }
+
+    @Test
+    @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
+    public void execute_fails_on_size_limit() throws Exception {
+        createDirectories(Path.of("target/test/CreateNewDataFilesTest/tmp/temp"));
+
+        mockTmpLookup();
+        var cmd = createCmd("scripts/search/data/binary/3files.zip", mockDatasetVersion(), 1000L, 500L);
+        var ctxt = mockCommandContext(mockSysConfig(true, 50L, MD5, 0));
+        try (var mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
+            mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");
+
+            assertThatThrownBy(() -> cmd.execute(ctxt))
+                .isInstanceOf(CommandException.class)
+                .hasMessage("This file size (462 B) exceeds the size limit of 50 B.");
+        }
+    }
+
+    @Test
+    @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
+    public void execute_loads_individual_files_from_uploaded_zip() throws Exception {
+        var tempDir = testDir.resolve("tmp/temp");
+        createDirectories(tempDir);
+
+        mockTmpLookup();
+        var cmd = createCmd("src/test/resources/own-cloud-downloads/greetings.zip", mockDatasetVersion(), 1000L, 500L);
+        var ctxt = mockCommandContext(mockSysConfig(false, 1000000L, MD5, 10));
+        try (MockedStatic<JhoveFileType> mockedStatic = Mockito.mockStatic(JhoveFileType.class)) {
+            mockedStatic.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");
+
+            // the test
+            var result = cmd.execute(ctxt);
+
+            assertThat(result.getErrors()).hasSize(0);
+            assertThat(result.getDataFiles().stream().map(dataFile ->
+                dataFile.getFileMetadata().getDirectoryLabel() + "/" + dataFile.getDisplayName()
+            )).containsExactlyInAnyOrder(
+                "DD-1576/goodbye.txt", "DD-1576/hello.txt"
+            );
+            var storageIds = result.getDataFiles().stream().map(DataFile::getStorageIdentifier).toList();
+            assertThat(tempDir.toFile().list())
+                .containsExactlyInAnyOrderElementsOf(storageIds);
+        }
+    }
+
+    @Test
+    @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "target/test/CreateNewDataFilesTest/tmp")
+    public void execute_rezips_sets_of_shape_files_from_uploaded_zip() throws Exception {
+        var tempDir = testDir.resolve("tmp/temp");
+        createDirectories(tempDir);
+
+        mockTmpLookup();
+        var cmd = createCmd("src/test/resources/own-cloud-downloads/shapes.zip", mockDatasetVersion(), 1000L, 500L);
+        var ctxt = mockCommandContext(mockSysConfig(false, 100000000L, MD5, 10));
+        try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) {
+            mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");
+
+            // the test
+            var result = cmd.execute(ctxt);
+
+            assertThat(result.getErrors()).hasSize(0);
+            assertThat(result.getDataFiles().stream().map(dataFile ->
+                (dataFile.getFileMetadata().getDirectoryLabel() + "/" + dataFile.getDisplayName())
+                    .replaceAll(".*temp/shp_[-0-9]*/", "")
+            )).containsExactlyInAnyOrder(
+                "dataDir/shape1.zip",
+                "dataDir/shape2/shape2",
+                "dataDir/shape2/shape2.pdf",
+                "dataDir/shape2/shape2.txt",
+                "dataDir/shape2/shape2.zip",
+                "dataDir/extra/shp_dictionary.xls",
+                "dataDir/extra/notes",
+                "dataDir/extra/README.MD"
+            );
+            var storageIds = result.getDataFiles().stream().map(DataFile::getStorageIdentifier).toList();
+            assertThat(tempDir.toFile().list())
+                .containsExactlyInAnyOrderElementsOf(storageIds);
+        }
+    }
+
+    private static @NotNull CreateNewDataFilesCommand createCmd(String name, DatasetVersion dsVersion, long allocatedQuotaLimit, long usedQuotaLimit) throws FileNotFoundException {
+        return new CreateNewDataFilesCommand(
+            Mockito.mock(DataverseRequest.class),
+            dsVersion,
+            new FileInputStream(name),
+            "example.zip",
+            "application/zip",
+            null,
+            new UploadSessionQuotaLimit(allocatedQuotaLimit, usedQuotaLimit),
+            "sha");
+    }
+
+    private static @NotNull CommandContext mockCommandContext(SystemConfig sysCfg) {
+        var ctxt = Mockito.mock(CommandContext.class);
+        Mockito.when(ctxt.systemConfig()).thenReturn(sysCfg);
+        return ctxt;
+    }
+
+    private static @NotNull SystemConfig mockSysConfig(boolean isStorageQuataEnforced, long maxFileUploadSizeForStore, DataFile.ChecksumType checksumType, int zipUploadFilesLimit) {
+        var sysCfg = Mockito.mock(SystemConfig.class);
+        Mockito.when(sysCfg.isStorageQuotasEnforced()).thenReturn(isStorageQuataEnforced);
+        Mockito.when(sysCfg.getMaxFileUploadSizeForStore(any())).thenReturn(maxFileUploadSizeForStore);
+        Mockito.when(sysCfg.getFileFixityChecksumAlgorithm()).thenReturn(checksumType);
+        Mockito.when(sysCfg.getZipUploadFilesLimit()).thenReturn(zipUploadFilesLimit);
+        return sysCfg;
+    }
+
+    private static void mockTmpLookup() {
+        JvmSettings mockFilesDirectory = Mockito.mock(JvmSettings.class);
+        Mockito.when(mockFilesDirectory.lookup()).thenReturn("/mocked/path");
+    }
+
+    private static @NotNull DatasetVersion mockDatasetVersion() {
+        var dsVersion = Mockito.mock(DatasetVersion.class);
+        Mockito.when(dsVersion.getDataset()).thenReturn(Mockito.mock(Dataset.class));
+        return dsVersion;
+    }
+
+}
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java
index 3c5b4797b0a..c4ee4547ed7 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/shapefile/ShapefileHandlerTest.java
@@ -63,22 +63,22 @@ private File createBlankFile(String filename) throws IOException {
         }
         return Files.createFile(tempFolder.resolve(filename)).toFile();
     }
-    
+
     private FileInputStream createZipReturnFilestream(List<String> file_names, String zipfile_name) throws IOException{
-        
+
         File zip_file_obj = this.createAndZipFiles(file_names, zipfile_name);
         if (zip_file_obj == null){
             return null;
         }
-        
+
         FileInputStream file_input_stream = new FileInputStream(zip_file_obj);
 
         return file_input_stream;
-        
+
     }
-    
+
     /*
-        Convenience class to create .zip file and return a FileInputStream
+        Convenience method to create .zip file and return a File
     
         @param List<String> file_names - List of filenames to add to .zip.  These names will be used to create 0 length files
         @param String zipfile_name - Name of .zip file to create
@@ -98,13 +98,13 @@ private File createAndZipFiles(List<String> file_names, String zipfile_name) thr
         }
         
         Path zip_file_obj = this.tempFolder.resolve(zipfile_name);
-        ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zip_file_obj.toFile()));
+        try (ZipOutputStream zip_stream = new ZipOutputStream(new FileOutputStream(zip_file_obj.toFile()))) {
 
-        // Iterate through File objects and add them to the ZipOutputStream
-        for (File file_obj : fileCollection) {
-             this.addToZipFile(file_obj.getName(), file_obj, zip_stream);
+            // Iterate through File objects and add them to the ZipOutputStream
+            for (File file_obj : fileCollection) {
+                this.addToZipFile(file_obj.getName(), file_obj, zip_stream);
+            }
         }
-
         /* -----------------------------------
         Cleanup: Delete single files that were added to .zip
         ----------------------------------- */
@@ -126,7 +126,7 @@ public void testCreateZippedNonShapefile() throws IOException{
         File zipfile_obj = createAndZipFiles(file_names, "not-quite-a-shape.zip");
         
         // Pass the .zip to the ShapefileHandler
-        ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(zipfile_obj));
+        ShapefileHandler shp_handler = new ShapefileHandler(zipfile_obj);
         shp_handler.DEBUG= true;
 
         // Contains shapefile?
@@ -157,7 +157,7 @@ public void testShapefileWithQpjAndQmd() throws IOException {
         File zipFile = createAndZipFiles(fileNames, "testShapeWithNewExtensions.zip");
 
         // Pass the zip to the ShapefileHandler
-        ShapefileHandler shpHandler = new ShapefileHandler(new FileInputStream(zipFile));
+        ShapefileHandler shpHandler = new ShapefileHandler(zipFile);
         shpHandler.DEBUG = true;
 
         // Check if it is recognized as a shapefile
@@ -191,7 +191,7 @@ public void testZippedTwoShapefiles() throws IOException{
         File zipfile_obj = createAndZipFiles(file_names, "two-shapes.zip");
         
         // Pass the .zip to the ShapefileHandler
-        ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(zipfile_obj));
+        ShapefileHandler shp_handler = new ShapefileHandler(zipfile_obj);
         shp_handler.DEBUG= true;
         
         assertTrue(shp_handler.containsShapefile(), "verify shapefile existance");
@@ -217,7 +217,7 @@ public void testZippedTwoShapefiles() throws IOException{
         // Rezip/Reorder the files
         File test_unzip_folder = Files.createDirectory(this.tempFolder.resolve("test_unzip")).toFile();
         //File test_unzip_folder = new File("/Users/rmp553/Desktop/blah");
-        shp_handler.rezipShapefileSets(new FileInputStream(zipfile_obj), test_unzip_folder );
+        shp_handler.rezipShapefileSets(test_unzip_folder );
         
    
         // Does the re-ordering do what we wanted?
@@ -244,7 +244,7 @@ public void testZippedShapefileWithExtraFiles() throws IOException{
         File zipfile_obj = createAndZipFiles(file_names, "shape-plus.zip");
 
         // Pass the .zip to the ShapefileHandler
-        ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(zipfile_obj));
+        ShapefileHandler shp_handler = new ShapefileHandler(zipfile_obj);
         shp_handler.DEBUG= true;
         
         assertTrue(shp_handler.containsShapefile(), "verify shapefile existance");
@@ -264,7 +264,7 @@ public void testZippedShapefileWithExtraFiles() throws IOException{
         
         File unzip2Folder = Files.createDirectory(this.tempFolder.resolve("test_unzip2")).toFile();
         // Rezip/Reorder the files
-        shp_handler.rezipShapefileSets(new FileInputStream(zipfile_obj), unzip2Folder);
+        shp_handler.rezipShapefileSets(unzip2Folder);
         //shp_handler.rezipShapefileSets(new FileInputStream(zipfile_obj), new File("/Users/rmp553/Desktop/blah"));
         
    
@@ -284,9 +284,9 @@ public void testZippedShapefileWithExtraFiles() throws IOException{
     }
 
     @Test
-    public void testHiddenFiles() {
+    public void testHiddenFiles() throws IOException {
         // test with shapefiles in hidden directory
-        ShapefileHandler shp_handler = new ShapefileHandler("src/test/resources/hiddenShapefiles.zip");
+        ShapefileHandler shp_handler = new ShapefileHandler(new File("src/test/resources/hiddenShapefiles.zip"));
         shp_handler.DEBUG= true;
         assertFalse(shp_handler.containsShapefile());
     }
diff --git a/src/test/resources/own-cloud-downloads/greetings.zip b/src/test/resources/own-cloud-downloads/greetings.zip
new file mode 100644
index 0000000000000000000000000000000000000000..6e166d385d18dce3b63eac9b742c40fea5dddae9
GIT binary patch
literal 679
zcmWIWW@gc400HJ?!$=VQ9||}a6c}7wbPY|-&GZ==WI$3lK>*kk4xl`1BC09ENT#Ic
z=cgo9rs|bclwdQ^4as<jQibyTqMQ`20B?2<_R{<Dyif%&8e!{)1XNoEk!;OK&B@8f
zW#=9wJHZN(tkkvUSkD8MhtUDvj7+-BxC0Cnd<+Z%3~wDlp-6o2K}>}OAjq@OSojYL
zK!~Xf3`-i3O+^b%Sg62g<luxk8)N|}tUz9aSpW(v>=vMe94rK2G;+uVc(bwT01W{F
uSm=P*%pd{?V6+$`gbz$lU;xy|4q~`6LHHoy0B=?{kkgofx?oOZU;qG5YmhPk

literal 0
HcmV?d00001

diff --git a/src/test/resources/own-cloud-downloads/shapes.zip b/src/test/resources/own-cloud-downloads/shapes.zip
new file mode 100644
index 0000000000000000000000000000000000000000..99d5f36c8952950869f77b3316e3c9d5b65c72fe
GIT binary patch
literal 4336
zcmbVOJ%|%Q7~NcQxm=^s5cLj=Ac7!wIg&!fLWtz9Sxy!$L{KETY)Je`Lb65tDHIgJ
zLR9cLEd)_38wCXm3vCn|wX(FZ6S1=s(fP9Tk}s2)u$yfj+1+{X&6oN1dyXBAEn0LP
zcXKBQ|H&(1WvoicD;4YQrHHkY9vLsk$g+vCxLvPZ(sHdZW}hn|?z}xV>}dN4*U{xd
zadkPnTGSc5%Sf@2{({U}v*Ec7&#`9`_a5bM&&ij8W*4au7mcRpc61JQL<Sb%p!YIY
zPdDFfq~&`+vwCJky;EzQuGH7Odb3e#U&waPX|1eltc<SYzTzJX&vr>)K(nu{%vYyY
zYPq>=rCik&B|9}*l)(z_;7#t*{dxH^FqDI8xuJ9L-0VQC;11lqpD#ig7}7yk=O7<7
z)}UJMat9B$q@O|=7|KDbeMaZtrP+a4!5z&0{<H#RU?>OTnCTqsiy3PmR&WRJ?IT~I
z3=HW&`!Bp6=V0hx`1WJ{Jd}aH1KNZ7<=m|HLSFmPJ2*WkfO$LVQL<jQ<aIT@!6@;;
z@!#h4Jbd$Kdz;h-nhngE444``5g+Vp*aWTMM%t%d>_$0(lNf1Lv|pM#W+P$+H*)#G
zmHjXPoXCj2+2&`A6;Yb)!$#~63;-uFV(7WYW+O_^CH7o93j@H(jA&bLf#$oslrz$)
z4YgkK`pGXa0Gz~#*Y$KA^U7>Qtl(#)`Tg$!7yvqUWNI;L&QFx4EGudKDp07xMs+?=
z48n1&l&AEpLDQA*82O5_4-y~cE1%o4o?PXQeeQmEq75J@0eRZO1|UKa2O<=o#)<U9
z6tWPIhbdKnvI+z!A&nCd2VIkwkZ^!JFR5yjOQ1%%N3S->G*m8}7$8qUDnF%;!2B}f
zfEUO!j>=D&A_RZ0fx!#p2}b3o#1NQY<`eJ&c|KA3DGvnZmtR8g0(tsS`DxmR;MdlN
zXAAI)gQCp}{`pbA68_u{kQ{~^nua^fQ+610IP*(`B!L_N<Vgbl(atosAwlUNfCRDv
z$`9mOL6xDQ99Bjq0nh_^5>RDm<c5~<=RD~9$i}NGLjyIejN$C^DXz-UxC}3&J;HoO
zt1>hU!^+4B+%J|mH+1m{?AY=6f`8k>T&15Wc@OBwLLZv-ABc6mJGYYl?}z}$o{p2n
KlvoX`mh~UjzepMY

literal 0
HcmV?d00001


From 97a38eadd9ecfee2a9eefaaec7dcf67bb35b211e Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Tue, 1 Oct 2024 12:05:31 -0400
Subject: [PATCH 045/137] Fix Deaccessioned edition.

---
 src/main/java/edu/harvard/iq/dataverse/Dataset.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 52cb7d6f2dc..3bcfbcb0d5e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -335,7 +335,7 @@ public DatasetVersion getLatestVersion() {
 
     public DatasetVersion getLatestVersionForCopy() {
         for (DatasetVersion testDsv : getVersions()) {
-            if (testDsv.isReleased() || testDsv.isArchived()) {
+            if (testDsv.isReleased() || testDsv.isArchived() || testDsv.isDeaccessioned()) {
                 return testDsv;
             }
         }

From 2aad633b85df87775ac978da1bcc35e612507d10 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Tue, 1 Oct 2024 14:36:27 -0400
Subject: [PATCH 046/137] Patch notes

---
 doc/release-notes/10901deaccessioned file edit fix.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/release-notes/10901deaccessioned file edit fix.md

diff --git a/doc/release-notes/10901deaccessioned file edit fix.md b/doc/release-notes/10901deaccessioned file edit fix.md
new file mode 100644
index 00000000000..db12b1fc978
--- /dev/null
+++ b/doc/release-notes/10901deaccessioned file edit fix.md	
@@ -0,0 +1 @@
+When a dataset was deaccessioned and was the only previous version it will cause an error when trying to update the files. 
\ No newline at end of file

From db1a066c411f9d25a782ecb55e99c2813aadf55a Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Tue, 1 Oct 2024 14:55:46 -0400
Subject: [PATCH 047/137] Added a signature of the method to receive
 includeDeaccessioned param

---
 src/main/java/edu/harvard/iq/dataverse/Dataset.java      | 9 +++++++--
 .../engine/command/impl/UpdateDatasetVersionCommand.java | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 3bcfbcb0d5e..40ed491a302 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -333,15 +333,20 @@ public DatasetVersion getLatestVersion() {
         return getVersions().get(0);
     }
 
-    public DatasetVersion getLatestVersionForCopy() {
+    public DatasetVersion getLatestVersionForCopy(boolean includeDeaccessioned) {
         for (DatasetVersion testDsv : getVersions()) {
-            if (testDsv.isReleased() || testDsv.isArchived() || testDsv.isDeaccessioned()) {
+            if (testDsv.isReleased() || testDsv.isArchived() 
+                || (testDsv.isDeaccessioned() && includeDeaccessioned)) {
                 return testDsv;
             }
         }
         return getVersions().get(0);
     }
 
+    public DatasetVersion getLatestVersionForCopy(){
+        return getLatestVersionForCopy(false);
+    }
+
     public List<DatasetVersion> getVersions() {
         return versions;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
index bb5f5a71e24..f501094ac92 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
@@ -115,7 +115,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
          */
         if(persistedVersion==null) {
             Long id = getDataset().getLatestVersion().getId();
-            persistedVersion = ctxt.datasetVersion().find(id!=null ? id: getDataset().getLatestVersionForCopy().getId());
+            persistedVersion = ctxt.datasetVersion().find(id!=null ? id : getDataset().getLatestVersionForCopy().getId());
         }
         
         //Will throw an IllegalCommandException if a system metadatablock is changed and the appropriate key is not supplied.

From c2e1e157ad96275f5ae0801320f7f3fcd8fe6ac6 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Tue, 1 Oct 2024 14:58:15 -0400
Subject: [PATCH 048/137] Missing call to the new method

---
 .../engine/command/impl/UpdateDatasetVersionCommand.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
index f501094ac92..dc8884405ef 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java
@@ -115,7 +115,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
          */
         if(persistedVersion==null) {
             Long id = getDataset().getLatestVersion().getId();
-            persistedVersion = ctxt.datasetVersion().find(id!=null ? id : getDataset().getLatestVersionForCopy().getId());
+            persistedVersion = ctxt.datasetVersion().find(id!=null ? id : getDataset().getLatestVersionForCopy(true).getId());
         }
         
         //Will throw an IllegalCommandException if a system metadatablock is changed and the appropriate key is not supplied.

From c46e0ce83969a6c678314a158205f1d133781167 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Wed, 2 Oct 2024 13:57:34 -0400
Subject: [PATCH 049/137] Test added

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index f52aa4fe9bd..e3ef11266ff 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -667,6 +667,57 @@ public void testCreatePublishDestroyDataset() {
         deleteDatasetResponse.prettyPrint();
         assertEquals(200, deleteDatasetResponse.getStatusCode());
 
+         // Start of deaccession test.
+
+        // Create Dataset for deaccession test.
+        Response deaccessionTestDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        deaccessionTestDataset.prettyPrint();
+        deaccessionTestDataset.then().assertThat().statusCode(CREATED.getStatusCode());
+        Integer deaccessionTestDatasetId = UtilIT.getDatasetIdFromResponse(deaccessionTestDataset);
+        
+        // File upload for deaccession test.
+        String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";   
+        Response uploadResponse = UtilIT.uploadFileViaNative(deaccessionTestDatasetId.toString(), pathToFile, apiToken);
+        uploadResponse.prettyPrint();
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Integer deaccessionTestFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+
+        // Publish Dataset for deaccession test.
+        Response deaccessionTestPublishResponse = UtilIT.publishDatasetViaNativeApi(deaccessionTestDatasetId, "major", apiToken);
+        deaccessionTestPublishResponse.prettyPrint();
+
+        // Deaccession Dataset for deaccession test.
+        Response deaccessionTestDatasetResponse = UtilIT.deaccessionDataset(deaccessionTestDatasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken);
+        deaccessionTestDatasetResponse.prettyPrint();
+        deaccessionTestDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Version check for deaccession test - Deaccessioned.
+        Response deaccessionTestVersions = UtilIT.getDatasetVersions(deaccessionTestDatasetId.toString(), apiToken);
+        deaccessionTestVersions.prettyPrint();
+        deaccessionTestVersions.then().assertThat()
+                .body("data[0].latestVersionPublishingState", equalTo("DEACCESSIONED"))
+                .statusCode(OK.getStatusCode());
+
+        // File deletion / Draft creation due deltigion check for deaccession test.
+        Response deaccessionTestDeleteFile =  UtilIT.deleteFileInDataset(deaccessionTestFileId, apiToken);
+        deaccessionTestDeleteFile.prettyPrint();
+
+        // Version check for deaccession test - Draft.
+        deaccessionTestVersions = UtilIT.getDatasetVersions(deaccessionTestDatasetId.toString(), apiToken);
+        deaccessionTestVersions.prettyPrint();
+        deaccessionTestVersions.then().assertThat()
+                .body("data[0].latestVersionPublishingState", equalTo("DRAFT"))
+                .statusCode(OK.getStatusCode());
+
+        // Deleting Dataset for deaccession test.
+        Response deaccessionTestDelete = UtilIT.destroyDataset(deaccessionTestDatasetId, apiToken);
+        deaccessionTestDelete.prettyPrint();
+        deaccessionTestDelete.then()
+                .assertThat()
+                .statusCode(OK.getStatusCode());
+        
+        //  End of deaccession test.
+
         Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
         deleteDataverseResponse.prettyPrint();
         assertEquals(200, deleteDataverseResponse.getStatusCode());

From 2f4a84cefbedc1e3cd0ded56c7b11a2ab5348a99 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Wed, 2 Oct 2024 13:58:08 -0400
Subject: [PATCH 050/137] Typo

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index e3ef11266ff..d6db7a6a6c6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -667,7 +667,7 @@ public void testCreatePublishDestroyDataset() {
         deleteDatasetResponse.prettyPrint();
         assertEquals(200, deleteDatasetResponse.getStatusCode());
 
-         // Start of deaccession test.
+        // Start of deaccession test.
 
         // Create Dataset for deaccession test.
         Response deaccessionTestDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);

From cb6e44f0d58f72aea6bae7476d85fa79ca2b31b7 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 2 Oct 2024 15:15:25 -0400
Subject: [PATCH 051/137] tweak release note #10875

---
 .../10886-update-to-conditions-to-display-image_url.md        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/10886-update-to-conditions-to-display-image_url.md b/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
index a7adda11840..6dfe8eb9f2d 100644
--- a/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
+++ b/doc/release-notes/10886-update-to-conditions-to-display-image_url.md
@@ -3,4 +3,6 @@ Search API (/api/search) responses for Datafiles include image_url for the thumb
 2. A Thumbnail is available for the Datafile
 3. If the Datafile is Restricted then the caller must have Download File Permission for the Datafile
 4. The Datafile is NOT actively embargoed
-5. The Datafile's retention has NOT expired
+5. The Datafile's retention period has NOT expired
+
+See also #10875 and #10886.

From e57f834558952d61f0efdd447dcdee49c7691927 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Wed, 2 Oct 2024 17:18:41 -0400
Subject: [PATCH 052/137] Remove of clone from the API file deletion.

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index d786aab35a8..c8bc8420944 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -343,10 +343,9 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP
             DataFile dataFile = findDataFileOrDie(fileIdOrPersistentId);
             FileMetadata fileToDelete = dataFile.getLatestFileMetadata();
             Dataset dataset = dataFile.getOwner();
-            DatasetVersion v = dataset.getOrCreateEditVersion();
             deletePhysicalFile = !dataFile.isReleased();
 
-            UpdateDatasetVersionCommand update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest,  Arrays.asList(fileToDelete), v);
+            UpdateDatasetVersionCommand update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest,  Arrays.asList(fileToDelete));
             update_cmd.setValidateLenient(true);
 
             try {

From 8accad72c693a2f92aa1a6198a80101623d07bcb Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Wed, 2 Oct 2024 18:13:51 -0400
Subject: [PATCH 053/137] Copy still needs to be created but not sent as a
 parameter to the command.

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index c8bc8420944..633d420c527 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -343,6 +343,7 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP
             DataFile dataFile = findDataFileOrDie(fileIdOrPersistentId);
             FileMetadata fileToDelete = dataFile.getLatestFileMetadata();
             Dataset dataset = dataFile.getOwner();
+            dataset.getOrCreateEditVersion();
             deletePhysicalFile = !dataFile.isReleased();
 
             UpdateDatasetVersionCommand update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest,  Arrays.asList(fileToDelete));

From 70a2651bc94a15e7a3bc5ad2423aa9edf2c73a99 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Wed, 2 Oct 2024 18:15:38 -0400
Subject: [PATCH 054/137] File upload assertion

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index d6db7a6a6c6..6a854b08023 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -701,7 +701,10 @@ public void testCreatePublishDestroyDataset() {
         // File deletion / Draft creation due deltigion check for deaccession test.
         Response deaccessionTestDeleteFile =  UtilIT.deleteFileInDataset(deaccessionTestFileId, apiToken);
         deaccessionTestDeleteFile.prettyPrint();
-
+        deaccessionTestDeleteFile
+                .then().assertThat()
+                .statusCode(OK.getStatusCode());
+                
         // Version check for deaccession test - Draft.
         deaccessionTestVersions = UtilIT.getDatasetVersions(deaccessionTestDatasetId.toString(), apiToken);
         deaccessionTestVersions.prettyPrint();

From 4e2bc1a0e5f37876d1c8defbeadfe7357c758fbf Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 3 Oct 2024 10:11:51 -0400
Subject: [PATCH 055/137] Update DatasetsIT.java

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 6a854b08023..d51ce010dab 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -667,7 +667,7 @@ public void testCreatePublishDestroyDataset() {
         deleteDatasetResponse.prettyPrint();
         assertEquals(200, deleteDatasetResponse.getStatusCode());
 
-        // Start of deaccession test.
+        // Start of test of deleting a file from a deaccessioned version.
 
         // Create Dataset for deaccession test.
         Response deaccessionTestDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
@@ -698,7 +698,7 @@ public void testCreatePublishDestroyDataset() {
                 .body("data[0].latestVersionPublishingState", equalTo("DEACCESSIONED"))
                 .statusCode(OK.getStatusCode());
 
-        // File deletion / Draft creation due deltigion check for deaccession test.
+        // File deletion / Draft creation due diligence check for deaccession test.
         Response deaccessionTestDeleteFile =  UtilIT.deleteFileInDataset(deaccessionTestFileId, apiToken);
         deaccessionTestDeleteFile.prettyPrint();
         deaccessionTestDeleteFile

From d50c484892519e342c2de440985fc554c92b3c4b Mon Sep 17 00:00:00 2001
From: jeromeroucou <jeromeroucou@users.noreply.github.com>
Date: Fri, 4 Oct 2024 17:40:46 +0200
Subject: [PATCH 056/137] Facets filter labels not translated in result block
 (#10158)

* indentation

* add comments et remove old "dead" code

* Add friendly name for value from filter query

* Add release note
---
 ...s-labels-not-translated-in-result-block.md |  7 +++
 .../search/SearchIncludeFragment.java         | 48 +++++++++++--------
 2 files changed, 34 insertions(+), 21 deletions(-)
 create mode 100644 doc/release-notes/9408-fix-facets-labels-not-translated-in-result-block.md

diff --git a/doc/release-notes/9408-fix-facets-labels-not-translated-in-result-block.md b/doc/release-notes/9408-fix-facets-labels-not-translated-in-result-block.md
new file mode 100644
index 00000000000..344859e2dbd
--- /dev/null
+++ b/doc/release-notes/9408-fix-facets-labels-not-translated-in-result-block.md
@@ -0,0 +1,7 @@
+## Fix facets filter labels not translated in result block
+
+On the main page, it's possible to filter results using search facets. If internationalization (i18n) has been activated in the Dataverse installation, allowing pages to be displayed in several languages, the facets are translated in the filter column. However, they aren't translated in the search results and remain in the default language, English.
+
+This version of Dataverse fix this, and includes internationalization in the facets visible in the search results section.
+
+For more information, see issue [#9408](https://github.com/IQSS/dataverse/issues/9408) and pull request [#10158](https://github.com/IQSS/dataverse/pull/10158)
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 4f3f6e46e48..9328dd03ca2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -34,6 +34,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.MissingResourceException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Logger;
@@ -1231,40 +1232,33 @@ public String getTypeFromFilterQuery(String filterQuery) {
     }
     
     public List<String> getFriendlyNamesFromFilterQuery(String filterQuery) {
-        
-        
-        if ((filterQuery == null)||
-            (datasetfieldFriendlyNamesBySolrField == null)||
-            (staticSolrFieldFriendlyNamesBySolrField==null)){
+
+        if ((filterQuery == null) ||
+                (datasetfieldFriendlyNamesBySolrField == null) ||
+                (staticSolrFieldFriendlyNamesBySolrField == null)) {
             return null;
         }
-        
-        if(!filterQuery.contains(":")) {
+
+        if (!filterQuery.contains(":")) {
             return null;
         }
-        
+
         int index = filterQuery.indexOf(":");
         String key = filterQuery.substring(0,index);
         String value = filterQuery.substring(index+1);
 
-        List<String> friendlyNames = new ArrayList<>();
+        // friendlyNames get 2 entries : key and value
+        List<String> friendlyNames = new ArrayList<>(2);
 
+        // Get dataset field friendly name from default ressource bundle file
         String datasetfieldFriendyName = datasetfieldFriendlyNamesBySolrField.get(key);
         if (datasetfieldFriendyName != null) {
             friendlyNames.add(datasetfieldFriendyName);
         } else {
+            // Get non dataset field friendly name from "staticSearchFields" ressource bundle file
             String nonDatasetSolrField = staticSolrFieldFriendlyNamesBySolrField.get(key);
             if (nonDatasetSolrField != null) {
                 friendlyNames.add(nonDatasetSolrField);
-            } else if (key.equals(SearchFields.PUBLICATION_STATUS)) {
-                /**
-                 * @todo Refactor this quick fix for
-                 * https://github.com/IQSS/dataverse/issues/618 . We really need
-                 * to get rid of all the reflection that's happening with
-                 * solrQueryResponse.getStaticSolrFieldFriendlyNamesBySolrField()
-                 * and
-                 */
-                friendlyNames.add("Publication Status");
             } else {
                 // meh. better than nuthin'
                 friendlyNames.add(key);
@@ -1276,9 +1270,13 @@ public List<String> getFriendlyNamesFromFilterQuery(String filterQuery) {
         String valueWithoutQuotes = noTrailingQuote;
 
         if (key.equals(SearchFields.METADATA_TYPES) && getDataverse() != null && getDataverse().getMetadataBlockFacets() != null) {
-            Optional<String> friendlyName = getDataverse().getMetadataBlockFacets().stream().filter(block -> block.getMetadataBlock().getName().equals(valueWithoutQuotes)).findFirst().map(block -> block.getMetadataBlock().getLocaleDisplayFacet());
+            Optional<String> friendlyName = getDataverse().getMetadataBlockFacets()
+                    .stream()
+                    .filter(block -> block.getMetadataBlock().getName().equals(valueWithoutQuotes))
+                    .findFirst()
+                    .map(block -> block.getMetadataBlock().getLocaleDisplayFacet());
             logger.fine(String.format("action=getFriendlyNamesFromFilterQuery key=%s value=%s friendlyName=%s", key, value, friendlyName));
-            if(friendlyName.isPresent()) {
+            if (friendlyName.isPresent()) {
                 friendlyNames.add(friendlyName.get());
                 return friendlyNames;
             }
@@ -1290,7 +1288,15 @@ public List<String> getFriendlyNamesFromFilterQuery(String filterQuery) {
             }
         }
 
-        friendlyNames.add(valueWithoutQuotes);
+        // Get value friendly name from default ressource bundle file
+        String valueFriendlyName;
+        try {
+            valueFriendlyName = BundleUtil.getStringFromPropertyFile(noTrailingQuote, "Bundle");
+        } catch (MissingResourceException e) {
+            valueFriendlyName = noTrailingQuote;
+        }
+
+        friendlyNames.add(valueFriendlyName);
         return friendlyNames;
     }
     

From 1a5ca4ba1b8f275fc44c7b1f67bf5d4509aa237f Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 4 Oct 2024 11:41:54 -0400
Subject: [PATCH 057/137] PR template, add hyphen to show issue title

---
 .github/PULL_REQUEST_TEMPLATE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index b57aa23fc0f..f2a779bbf21 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -2,7 +2,7 @@
 
 **Which issue(s) this PR closes**:
 
-Closes #
+- Closes #
 
 **Special notes for your reviewer**:
 

From 7d3ab225af18650b67d9e018bb37806229f69bf1 Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Mon, 7 Oct 2024 09:51:24 +0200
Subject: [PATCH 058/137] open zip once and reuse list of entries

---
 .../command/impl/CreateNewDataFilesCommand.java    | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index e543606e039..76939751899 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -317,7 +317,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                      */
 
                     try (var zipFile = openZipFile(tempFile, charset)) {
-                        for (var entry : filteredZipEntries(zipFile)) {
+                        var zipEntries = filteredZipEntries(zipFile);
+                        for (var entry : zipEntries) {
                             logger.fine("inside first zip pass; this entry: " + entry.getName());
                             numberOfUnpackableFiles++;
                             if (numberOfUnpackableFiles > fileNumberLimit) {
@@ -349,15 +350,12 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                                 }
                             }
                         }
-                    }
+                        // OK we're still here - that means we can proceed unzipping.
 
-                    // OK we're still here - that means we can proceed unzipping. 
-                    
-                    // reset:
-                    combinedUnzippedFileSize = 0L;
+                        // reset:
+                        combinedUnzippedFileSize = 0L;
 
-                    try (var zipFile = openZipFile(tempFile, charset)) {
-                        for (var entry : filteredZipEntries(zipFile)) {
+                        for (var entry : zipEntries) {
                             if (datafiles.size() > fileNumberLimit) {
                                 logger.warning("Zip upload - too many files.");
                                 warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit

From 7cc81e0ec33ba854ffb581f0da7cf0019b2e6798 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Wed, 25 Sep 2024 15:13:55 -0400
Subject: [PATCH 059/137] add isRelease field to isPartOf

---
 .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java    | 1 +
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 7 +++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 34c8fc5c6a6..1bdee48b14d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -340,6 +340,7 @@ private static JsonObjectBuilder addEmbeddedOwnerObject(DvObject dvo, JsonObject
             ownerObject.add("type", "DATAVERSE");
             Dataverse in = (Dataverse) dvo;
             ownerObject.add("identifier", in.getAlias());
+            ownerObject.add("isReleased", in.isReleased());
         }
 
         if (dvo.isInstanceofDataset()) {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 8e38a96fc97..93f1024ae7a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -2134,8 +2134,11 @@ public void testGetDatasetOwners() {
 
         Response getDatasetWithOwners = UtilIT.getDatasetWithOwners(persistentId, apiToken, true);
         getDatasetWithOwners.prettyPrint();
-        getDatasetWithOwners.then().assertThat().body("data.isPartOf.identifier", equalTo(dataverseAlias));       
-        
+        getDatasetWithOwners.then().assertThat().body("data.isPartOf.identifier", equalTo(dataverseAlias));
+        getDatasetWithOwners.then().assertThat().body("data.isPartOf.isReleased", equalTo(false));
+        getDatasetWithOwners.then().assertThat().body("data.isPartOf.isPartOf.identifier", equalTo("root"));
+        getDatasetWithOwners.then().assertThat().body("data.isPartOf.isPartOf.isReleased", equalTo(true));
+
         Response destroyDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken);
         assertEquals(200, destroyDatasetResponse.getStatusCode());
 

From 36d15a501f50a46e940e6cc54c5cdf7619f4027d Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 31 Oct 2023 11:13:35 +0100
Subject: [PATCH 060/137] Reorder modifiers to static final

---
 .../iq/dataverse/DatasetFieldConstant.java    | 313 +++++++++---------
 .../harvard/iq/dataverse/WidgetWrapper.java   |   4 +-
 .../AbstractOAuth2AuthenticationProvider.java |   2 +-
 .../providers/oauth2/impl/OrcidOAuth2AP.java  |   2 +-
 .../engine/command/DataverseRequest.java      |   4 +-
 .../iq/dataverse/util/ShapefileHandler.java   |  12 +-
 .../iq/dataverse/util/SystemConfig.java       |   4 +-
 .../PasswordValidatorServiceBean.java         |   2 +-
 .../AuthenticationProviderTest.java           |   2 +-
 9 files changed, 172 insertions(+), 173 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java
index abb812d1ba3..71e339a6fca 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java
@@ -16,165 +16,164 @@
 @Named("dfc")
 @Dependent
 public class DatasetFieldConstant implements java.io.Serializable  {
-    
-    public final static String publication = "publication";
-    public final static String otherId = "otherId";
-    public final static String author =  "author";
-    public final static String authorFirstName =  "authorFirstName";
-    public final static String authorLastName =  "authorLastName";
-    public final static String producer = "producer";
-    public final static String software = "software";
-    public final static String grantNumber = "grantNumber";
-    public final static String distributor = "distributor";
-    public final static String datasetContact = "datasetContact";
-    public final static String datasetContactEmail = "datasetContactEmail";
-    public final static String datasetContactName = "datasetContactName";
-    public final static String datasetContactAffiliation = "datasetContactAffiliation";
-    public final static String series = "series";
-    public final static String datasetVersion = "datasetVersion";
-    
-    public final static String description = "dsDescription";
-    public final static String keyword = "keyword";
-    public final static String topicClassification = "topicClassification";
-    public final static String geographicBoundingBox = "geographicBoundingBox";
-    
-    public final static String note = "note";
-    
-    public final static String publicationRelationType = "publicationRelationType";
-    public final static String publicationCitation = "publicationCitation";
-    public final static String publicationIDType = "publicationIDType";
-    public final static String publicationIDNumber = "publicationIDNumber";
-    public final static String publicationURL = "publicationURL";
-    public final static String publicationReplicationData = "publicationReplicationData";
-    
-    
-    public final static String title = "title";
-    public final static String subTitle="subtitle"; //SEK 6-7-2016 to match what is in DB
-    public final static String alternativeTitle="alternativeTitle"; //missing from class
-    public final static String datasetId = "datasetId";
-    public final static String authorName ="authorName";
-    public final static String authorAffiliation = "authorAffiliation";
-    public final static String authorIdType = "authorIdentifierScheme";
-    public final static String authorIdValue = "authorIdentifier";
-    public final static String otherIdValue="otherIdValue";
-    public final static String otherIdAgency= "otherIdAgency";
-    
-    public final static String producerName="producerName";
-    public final static String producerURL="producerURL";
-    public final static String producerLogo="producerLogoURL";
-    public final static String producerAffiliation="producerAffiliation";
-    public final static String producerAbbreviation= "producerAbbreviation";
-    public final static String productionDate="productionDate";
-    public final static String productionPlace="productionPlace";
-    public final static String softwareName="softwareName";
-    public final static String softwareVersion="softwareVersion";
-    public final static String fundingAgency="fundingAgency";
-    public final static String grantNumberValue="grantNumberValue";
-    public final static String grantNumberAgency="grantNumberAgency";
-    public final static String distributorName="distributorName";
-    public final static String distributorURL="distributorURL";
-    public final static String distributorLogo="distributorLogoURL";
-    public final static String distributionDate="distributionDate";
-    public final static String distributorContactName="distributorContactName";
-    public final static String distributorContactAffiliation="distributorContactAffiliation";
-    public final static String distributorContactEmail="distributorContactEmail";
-    public final static String distributorAffiliation="distributorAffiliation";
-    public final static String distributorAbbreviation="distributorAbbreviation";
-
-    public final static String contributor="contributor"; //SEK added for Dublin Core 6/22
-    public final static String contributorType="contributorType";
-    public final static String contributorName="contributorName";
-
-    public final static String depositor="depositor";
-    public final static String dateOfDeposit="dateOfDeposit";
-    public final static String seriesName="seriesName";
-    public final static String seriesInformation="seriesInformation";
-    public final static String datasetVersionValue="datasetVersionValue";
-    public final static String versionDate="versionDate";
-    public final static String keywordValue="keywordValue";
-    public final static String keywordTermURI="keywordTermURI";
-    public final static String keywordVocab="keywordVocabulary";
-    public final static String keywordVocabURI="keywordVocabularyURI";
-    public final static String topicClassValue="topicClassValue";
-    public final static String topicClassVocab="topicClassVocab";
-    public final static String topicClassVocabURI="topicClassVocabURI";
-    public final static String descriptionText="dsDescriptionValue";
-    public final static String descriptionDate="dsDescriptionDate";
-    public final static String timePeriodCovered="timePeriodCovered"; // SEK added 6/13/2016
-    public final static String timePeriodCoveredStart="timePeriodCoveredStart";
-    public final static String timePeriodCoveredEnd="timePeriodCoveredEnd";
-    public final static String dateOfCollection="dateOfCollection"; // SEK added 6/13/2016
-    public final static String dateOfCollectionStart="dateOfCollectionStart";
-    public final static String dateOfCollectionEnd="dateOfCollectionEnd";
-    public final static String country="country";
-    public final static String geographicCoverage="geographicCoverage";
-    public final static String otherGeographicCoverage="otherGeographicCoverage";
-    public final static String city="city";  // SEK added 6/13/2016
-    public final static String state="state";  // SEK added 6/13/2016
-    public final static String geographicUnit="geographicUnit";
-    public final static String westLongitude="westLongitude";
-    public final static String eastLongitude="eastLongitude";
-    public final static String northLatitude="northLatitude";
-    public final static String southLatitude="southLatitude";
-    public final static String unitOfAnalysis="unitOfAnalysis";
-    public final static String universe="universe";
-    public final static String kindOfData="kindOfData";
-    public final static String timeMethod="timeMethod";
-    public final static String dataCollector="dataCollector";
-    public final static String collectorTraining="collectorTraining";
-    public final static String frequencyOfDataCollection="frequencyOfDataCollection";
-    public final static String samplingProcedure="samplingProcedure";
-    public final static String targetSampleSize = "targetSampleSize";
-    public final static String targetSampleActualSize = "targetSampleActualSize";
-    public final static String targetSampleSizeFormula = "targetSampleSizeFormula";
-    public final static String deviationsFromSampleDesign="deviationsFromSampleDesign";
-    public final static String collectionMode="collectionMode";
-    public final static String researchInstrument="researchInstrument";
-    public final static String dataSources="dataSources";
-    public final static String originOfSources="originOfSources";
-    public final static String characteristicOfSources="characteristicOfSources";
-    public final static String accessToSources="accessToSources";
-    public final static String dataCollectionSituation="dataCollectionSituation";
-    public final static String actionsToMinimizeLoss="actionsToMinimizeLoss";
-    public final static String controlOperations="controlOperations";
-    public final static String weighting="weighting";
-    public final static String cleaningOperations="cleaningOperations";
-    public final static String datasetLevelErrorNotes="datasetLevelErrorNotes";
-    public final static String responseRate="responseRate";
-    public final static String samplingErrorEstimates="samplingErrorEstimates";
-    
-    public final static String socialScienceNotes = "socialScienceNotes";
-    public final static String socialScienceNotesType = "socialScienceNotesType";
-    public final static String socialScienceNotesSubject = "socialScienceNotesSubject";
-    public final static String socialScienceNotesText = "socialScienceNotesText";
-
-    public final static String otherDataAppraisal="otherDataAppraisal";
-    public final static String placeOfAccess="placeOfAccess";
-    public final static String originalArchive="originalArchive";
-    public final static String availabilityStatus="availabilityStatus";
-    public final static String collectionSize="collectionSize";
-    public final static String datasetCompletion="datasetCompletion";
-    public final static String numberOfFiles="numberOfFiles";
-    public final static String confidentialityDeclaration="confidentialityDeclaration";
-    public final static String specialPermissions="specialPermissions";
-    public final static String restrictions="restrictions";
+
+    public static final String publication = "publication";
+    public static final String otherId = "otherId";
+    public static final String author =  "author";
+    public static final String authorFirstName =  "authorFirstName";
+    public static final String authorLastName =  "authorLastName";
+    public static final String producer = "producer";
+    public static final String software = "software";
+    public static final String grantNumber = "grantNumber";
+    public static final String distributor = "distributor";
+    public static final String datasetContact = "datasetContact";
+    public static final String datasetContactEmail = "datasetContactEmail";
+    public static final String datasetContactName = "datasetContactName";
+    public static final String datasetContactAffiliation = "datasetContactAffiliation";
+    public static final String series = "series";
+    public static final String datasetVersion = "datasetVersion";
+
+    public static final String description = "dsDescription";
+    public static final String keyword = "keyword";
+    public static final String topicClassification = "topicClassification";
+    public static final String geographicBoundingBox = "geographicBoundingBox";
+
+    public static final String note = "note";
+
+    public static final String publicationRelationType = "publicationRelationType";
+    public static final String publicationCitation = "publicationCitation";
+    public static final String publicationIDType = "publicationIDType";
+    public static final String publicationIDNumber = "publicationIDNumber";
+    public static final String publicationURL = "publicationURL";
+    public static final String publicationReplicationData = "publicationReplicationData";
+
+    public static final String title = "title";
+    public static final String subTitle="subtitle"; //SEK 6-7-2016 to match what is in DB
+    public static final String alternativeTitle="alternativeTitle"; //missing from class
+    public static final String datasetId = "datasetId";
+    public static final String authorName ="authorName";
+    public static final String authorAffiliation = "authorAffiliation";
+    public static final String authorIdType = "authorIdentifierScheme";
+    public static final String authorIdValue = "authorIdentifier";
+    public static final String otherIdValue="otherIdValue";
+    public static final String otherIdAgency= "otherIdAgency";
+
+    public static final String producerName="producerName";
+    public static final String producerURL="producerURL";
+    public static final String producerLogo="producerLogoURL";
+    public static final String producerAffiliation="producerAffiliation";
+    public static final String producerAbbreviation= "producerAbbreviation";
+    public static final String productionDate="productionDate";
+    public static final String productionPlace="productionPlace";
+    public static final String softwareName="softwareName";
+    public static final String softwareVersion="softwareVersion";
+    public static final String fundingAgency="fundingAgency";
+    public static final String grantNumberValue="grantNumberValue";
+    public static final String grantNumberAgency="grantNumberAgency";
+    public static final String distributorName="distributorName";
+    public static final String distributorURL="distributorURL";
+    public static final String distributorLogo="distributorLogoURL";
+    public static final String distributionDate="distributionDate";
+    public static final String distributorContactName="distributorContactName";
+    public static final String distributorContactAffiliation="distributorContactAffiliation";
+    public static final String distributorContactEmail="distributorContactEmail";
+    public static final String distributorAffiliation="distributorAffiliation";
+    public static final String distributorAbbreviation="distributorAbbreviation";
+
+    public static final String contributor="contributor"; //SEK added for Dublin Core 6/22
+    public static final String contributorType="contributorType";
+    public static final String contributorName="contributorName";
+
+    public static final String depositor="depositor";
+    public static final String dateOfDeposit="dateOfDeposit";
+    public static final String seriesName="seriesName";
+    public static final String seriesInformation="seriesInformation";
+    public static final String datasetVersionValue="datasetVersionValue";
+    public static final String versionDate="versionDate";
+    public static final String keywordValue="keywordValue";
+    public static final String keywordTermURI="keywordTermURI";
+    public static final String keywordVocab="keywordVocabulary";
+    public static final String keywordVocabURI="keywordVocabularyURI";
+    public static final String topicClassValue="topicClassValue";
+    public static final String topicClassVocab="topicClassVocab";
+    public static final String topicClassVocabURI="topicClassVocabURI";
+    public static final String descriptionText="dsDescriptionValue";
+    public static final String descriptionDate="dsDescriptionDate";
+    public static final String timePeriodCovered="timePeriodCovered"; // SEK added 6/13/2016
+    public static final String timePeriodCoveredStart="timePeriodCoveredStart";
+    public static final String timePeriodCoveredEnd="timePeriodCoveredEnd";
+    public static final String dateOfCollection="dateOfCollection"; // SEK added 6/13/2016
+    public static final String dateOfCollectionStart="dateOfCollectionStart";
+    public static final String dateOfCollectionEnd="dateOfCollectionEnd";
+    public static final String country="country";
+    public static final String geographicCoverage="geographicCoverage";
+    public static final String otherGeographicCoverage="otherGeographicCoverage";
+    public static final String city="city";  // SEK added 6/13/2016
+    public static final String state="state";  // SEK added 6/13/2016
+    public static final String geographicUnit="geographicUnit";
+    public static final String westLongitude="westLongitude";
+    public static final String eastLongitude="eastLongitude";
+    public static final String northLatitude="northLatitude";
+    public static final String southLatitude="southLatitude";
+    public static final String unitOfAnalysis="unitOfAnalysis";
+    public static final String universe="universe";
+    public static final String kindOfData="kindOfData";
+    public static final String timeMethod="timeMethod";
+    public static final String dataCollector="dataCollector";
+    public static final String collectorTraining="collectorTraining";
+    public static final String frequencyOfDataCollection="frequencyOfDataCollection";
+    public static final String samplingProcedure="samplingProcedure";
+    public static final String targetSampleSize = "targetSampleSize";
+    public static final String targetSampleActualSize = "targetSampleActualSize";
+    public static final String targetSampleSizeFormula = "targetSampleSizeFormula";
+    public static final String deviationsFromSampleDesign="deviationsFromSampleDesign";
+    public static final String collectionMode="collectionMode";
+    public static final String researchInstrument="researchInstrument";
+    public static final String dataSources="dataSources";
+    public static final String originOfSources="originOfSources";
+    public static final String characteristicOfSources="characteristicOfSources";
+    public static final String accessToSources="accessToSources";
+    public static final String dataCollectionSituation="dataCollectionSituation";
+    public static final String actionsToMinimizeLoss="actionsToMinimizeLoss";
+    public static final String controlOperations="controlOperations";
+    public static final String weighting="weighting";
+    public static final String cleaningOperations="cleaningOperations";
+    public static final String datasetLevelErrorNotes="datasetLevelErrorNotes";
+    public static final String responseRate="responseRate";
+    public static final String samplingErrorEstimates="samplingErrorEstimates";
+
+    public static final String socialScienceNotes = "socialScienceNotes";
+    public static final String socialScienceNotesType = "socialScienceNotesType";
+    public static final String socialScienceNotesSubject = "socialScienceNotesSubject";
+    public static final String socialScienceNotesText = "socialScienceNotesText";
+
+    public static final String otherDataAppraisal="otherDataAppraisal";
+    public static final String placeOfAccess="placeOfAccess";
+    public static final String originalArchive="originalArchive";
+    public static final String availabilityStatus="availabilityStatus";
+    public static final String collectionSize="collectionSize";
+    public static final String datasetCompletion="datasetCompletion";
+    public static final String numberOfFiles="numberOfFiles";
+    public static final String confidentialityDeclaration="confidentialityDeclaration";
+    public static final String specialPermissions="specialPermissions";
+    public static final String restrictions="restrictions";
     @Deprecated
     //Doesn't appear to be used and is not datasetContact
-    public final static String contact="contact";
-    public final static String citationRequirements="citationRequirements";
-    public final static String depositorRequirements="depositorRequirements";
-    public final static String conditions="conditions";
-    public final static String disclaimer="disclaimer";
-    public final static String relatedMaterial="relatedMaterial";
-    //public final static String replicationFor="replicationFor";
-    //public final static String relatedPublications="relatedPublications";
-    public final static String relatedDatasets="relatedDatasets";
-    public final static String otherReferences="otherReferences";
-    public final static String notesText="notesText";
-    public final static String language="language";
-    public final static String noteInformationType="noteInformationType";
-    public final static String notesInformationSubject="notesInformationSubject";
-    public final static String subject="subject";
+    public static final String contact="contact";
+    public static final String citationRequirements="citationRequirements";
+    public static final String depositorRequirements="depositorRequirements";
+    public static final String conditions="conditions";
+    public static final String disclaimer="disclaimer";
+    public static final String relatedMaterial="relatedMaterial";
+    //public static final String replicationFor="replicationFor";
+    //public static final String relatedPublications="relatedPublications";
+    public static final String relatedDatasets="relatedDatasets";
+    public static final String otherReferences="otherReferences";
+    public static final String notesText="notesText";
+    public static final String language="language";
+    public static final String noteInformationType="noteInformationType";
+    public static final String notesInformationSubject="notesInformationSubject";
+    public static final String subject="subject";
     /*
      * The following getters are needed so we can use them as properties in JSP 
      */
diff --git a/src/main/java/edu/harvard/iq/dataverse/WidgetWrapper.java b/src/main/java/edu/harvard/iq/dataverse/WidgetWrapper.java
index a8ea5fabde4..c51903e2ed4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/WidgetWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/WidgetWrapper.java
@@ -18,8 +18,8 @@
 @Named
 public class WidgetWrapper implements java.io.Serializable {
 
-    private final static String WIDGET_PARAMETER = "widget";
-    private final static char WIDGET_SEPARATOR = '@';
+    private static final String WIDGET_PARAMETER = "widget";
+    private static final char WIDGET_SEPARATOR = '@';
     
     private Boolean widgetView;
     private String widgetHome;
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java
index 7fd7bf3e885..a6b7c1b9d49 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java
@@ -30,7 +30,7 @@
  */
 public abstract class AbstractOAuth2AuthenticationProvider implements AuthenticationProvider {
 
-    final static Logger logger = Logger.getLogger(AbstractOAuth2AuthenticationProvider.class.getName());
+    static final Logger logger = Logger.getLogger(AbstractOAuth2AuthenticationProvider.class.getName());
 
     protected static class ParsedUserResponse {
         public final AuthenticatedUserDisplayInfo displayInfo;
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java
index 089ca40e164..323c78ab47a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java
@@ -49,7 +49,7 @@
  */
 public class OrcidOAuth2AP extends AbstractOAuth2AuthenticationProvider {
     
-    final static Logger logger = Logger.getLogger(OrcidOAuth2AP.class.getName());
+    static final Logger logger = Logger.getLogger(OrcidOAuth2AP.class.getName());
 
     public static final String PROVIDER_ID_PRODUCTION = "orcid";
     public static final String PROVIDER_ID_SANDBOX = "orcid-sandbox";
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/DataverseRequest.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/DataverseRequest.java
index d792b616a0c..4d3ec2842a1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/DataverseRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/DataverseRequest.java
@@ -26,9 +26,9 @@ public class DataverseRequest {
     private final String invocationId;
     private final HttpServletRequest httpServletRequest;
     
-    private final static String undefined = "0.0.0.0";
+    private static final String undefined = "0.0.0.0";
     
-    private final static String MDKEY_PREFIX="mdkey.";
+    private static final String MDKEY_PREFIX="mdkey.";
     
     private static final Logger logger = Logger.getLogger(DataverseRequest.class.getName());
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
index f1440cc3c02..bb916cc3906 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
@@ -68,12 +68,12 @@ public class ShapefileHandler{
     private static final Logger logger = Logger.getLogger(ShapefileHandler.class.getCanonicalName());
 
     // Reference for these extensions: http://en.wikipedia.org/wiki/Shapefile
-    public final static String SHAPEFILE_FILE_TYPE = "application/zipped-shapefile";
-    public final static String SHAPEFILE_FILE_TYPE_FRIENDLY_NAME = "Shapefile as ZIP Archive";
-    public final static List<String> SHAPEFILE_MANDATORY_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj");
-    public final static String SHP_XML_EXTENSION = "shp.xml";
-    public final static String BLANK_EXTENSION = "__PLACEHOLDER-FOR-BLANK-EXTENSION__";
-    public final static List<String> SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", "qpj", "qmd", SHP_XML_EXTENSION);
+    public static final String SHAPEFILE_FILE_TYPE = "application/zipped-shapefile";
+    public static final String SHAPEFILE_FILE_TYPE_FRIENDLY_NAME = "Shapefile as ZIP Archive";
+    public static final List<String> SHAPEFILE_MANDATORY_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj");
+    public static final String SHP_XML_EXTENSION = "shp.xml";
+    public static final String BLANK_EXTENSION = "__PLACEHOLDER-FOR-BLANK-EXTENSION__";
+    public static final List<String> SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", "qpj", "qmd", SHP_XML_EXTENSION);
     
     public boolean DEBUG = false;
         
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 60967b13131..2e1f7cb5cb3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -87,8 +87,8 @@ public class SystemConfig {
     private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE = 3000000L; // 3 MB
     private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_PDF = 1000000L; // 1 MB
     
-    public final static String DEFAULTCURATIONLABELSET = "DEFAULT";
-    public final static String CURATIONLABELSDISABLED = "DISABLED";
+    public static final String DEFAULTCURATIONLABELSET = "DEFAULT";
+    public static final String CURATIONLABELSDISABLED = "DISABLED";
     
     public String getVersion() {
         return getVersion(false);
diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java
index 41e7f1b8b22..92eafa5b856 100644
--- a/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java
@@ -84,7 +84,7 @@ private enum ValidatorTypes {
     }
 
     @SuppressWarnings("unchecked")
-    private final static LinkedHashMap<ValidatorTypes, PasswordValidator> validators = new LinkedHashMap(2);
+    private static final LinkedHashMap<ValidatorTypes, PasswordValidator> validators = new LinkedHashMap(2);
     private int goodStrength;
     private int maxLength;
     private int minLength;
diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java
index eac9a605c9e..d4d7b6fa69d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/AuthenticationProviderTest.java
@@ -15,7 +15,7 @@
 
 public class AuthenticationProviderTest {
 
-    private final static String[] authProviders = {"null", "builtin", "github", "google", "orcid", "orcid-sandbox", "shib"};
+    private static final String[] authProviders = {"null", "builtin", "github", "google", "orcid", "orcid-sandbox", "shib"};
     private static Map<String, String> bundleTestMap;
 
     @BeforeAll

From ba35f992a30cc63f14f8144bb9fa611719337134 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 31 Oct 2023 11:14:41 +0100
Subject: [PATCH 061/137] Use Map type and diamond operator

---
 .../dataverse/validation/PasswordValidatorServiceBean.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java
index 92eafa5b856..bbe7d135e0f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/validation/PasswordValidatorServiceBean.java
@@ -13,6 +13,7 @@
 import java.util.Date;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -83,8 +84,7 @@ private enum ValidatorTypes {
         GoodStrengthValidator, StandardValidator
     }
 
-    @SuppressWarnings("unchecked")
-    private static final LinkedHashMap<ValidatorTypes, PasswordValidator> validators = new LinkedHashMap(2);
+    private static final Map<ValidatorTypes, PasswordValidator> validators = new LinkedHashMap<>(2);
     private int goodStrength;
     private int maxLength;
     private int minLength;
@@ -100,7 +100,7 @@ private enum ValidatorTypes {
     public PasswordValidatorServiceBean() {
         final Properties properties = PropertiesMessageResolver.getDefaultProperties();
         properties.setProperty(GoodStrengthRule.ERROR_CODE_GOODSTRENGTH, GoodStrengthRule.ERROR_MESSAGE_GOODSTRENGTH);
-         messageResolver = new PropertiesMessageResolver(properties);
+        messageResolver = new PropertiesMessageResolver(properties);
     }
 
     public PasswordValidatorServiceBean(List<CharacterRule> characterRules) {

From a32b9ed685f3973044916c53dafecd136eecc942 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 31 Oct 2023 11:15:52 +0100
Subject: [PATCH 062/137] Replace deprecated constructors with valueOf(String)

---
 .../edu/harvard/iq/dataverse/util/SystemConfig.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 2e1f7cb5cb3..434b3bd8f8f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -473,7 +473,7 @@ public Integer getSearchHighlightFragmentSize() {
         String fragSize = settingsService.getValueForKey(SettingsServiceBean.Key.SearchHighlightFragmentSize);
         if (fragSize != null) {
             try {
-                return new Integer(fragSize);
+                return Integer.valueOf(fragSize);
             } catch (NumberFormatException nfe) {
                 logger.info("Could not convert " + SettingsServiceBean.Key.SearchHighlightFragmentSize + " to int: " + nfe);
             }
@@ -490,7 +490,7 @@ public long getTabularIngestSizeLimit() {
         
         if (limitEntry != null) {
             try {
-                Long sizeOption = new Long(limitEntry);
+                Long sizeOption = Long.valueOf(limitEntry);
                 return sizeOption;
             } catch (NumberFormatException nfe) {
                 logger.warning("Invalid value for TabularIngestSizeLimit option? - " + limitEntry);
@@ -515,7 +515,7 @@ public long getTabularIngestSizeLimit(String formatName) {
                 
         if (limitEntry != null) {
             try {
-                Long sizeOption = new Long(limitEntry);
+                Long sizeOption = Long.valueOf(limitEntry);
                 return sizeOption;
             } catch (NumberFormatException nfe) {
                 logger.warning("Invalid value for TabularIngestSizeLimit:" + formatName + "? - " + limitEntry );
@@ -1061,7 +1061,7 @@ public long getDatasetValidationSizeLimit() {
 
         if (limitEntry != null) {
             try {
-                Long sizeOption = new Long(limitEntry);
+                Long sizeOption = Long.valueOf(limitEntry);
                 return sizeOption;
             } catch (NumberFormatException nfe) {
                 logger.warning("Invalid value for DatasetValidationSizeLimit option? - " + limitEntry);
@@ -1076,7 +1076,7 @@ public long getFileValidationSizeLimit() {
 
         if (limitEntry != null) {
             try {
-                Long sizeOption = new Long(limitEntry);
+                Long sizeOption = Long.valueOf(limitEntry);
                 return sizeOption;
             } catch (NumberFormatException nfe) {
                 logger.warning("Invalid value for FileValidationSizeLimit option? - " + limitEntry);

From 8c4cd4c59be741b797e04dc59574522dacabfb72 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 11 Oct 2024 14:42:22 -0400
Subject: [PATCH 063/137] fix if statement body

---
 .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java
index a74a9f34bc9..8199b7d9c9f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java
@@ -1317,8 +1317,8 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject, boolean
                             }
                             if (StringUtils.isNotBlank(softwareName)) {
                                 if (StringUtils.isNotBlank(softwareVersion)) {
+                                    softwareName = softwareName + ", " + softwareVersion;
                                 }
-                                softwareName = softwareName + ", " + softwareVersion;
                                 descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten);
                                 XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, softwareName);
                             }

From f1380b1b760a3d1d90bc3c3e1de83cec142b1d35 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 11 Oct 2024 15:05:59 -0400
Subject: [PATCH 064/137] release note

---
 doc/release-notes/10919-minor-DataCiteXML-bugfix.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/release-notes/10919-minor-DataCiteXML-bugfix.md

diff --git a/doc/release-notes/10919-minor-DataCiteXML-bugfix.md b/doc/release-notes/10919-minor-DataCiteXML-bugfix.md
new file mode 100644
index 00000000000..4fa0c1142b1
--- /dev/null
+++ b/doc/release-notes/10919-minor-DataCiteXML-bugfix.md
@@ -0,0 +1 @@
+A minor bug fix was made to avoid sending a useless ", null" in the DataCiteXML sent to DataCite and in the DataCite export when a dataset has a metadata entry for "Software Name" and no entry for "Software Version". The bug fix will update datasets upon publication. Anyone with existing published datasets with this problem can be fixed by [pushing updated metadata to DataCite for affected datasets](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) and [re-exporting the dataset metadata](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) or by following steps 9 and 10 in the v6.4 release notes to update and re-export all datasets.

From 212df5e7f9c7100bdc58b9179b3827c9b0c2a342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= <pkiraly@gwdg.de>
Date: Sat, 12 Oct 2024 09:25:04 +0200
Subject: [PATCH 065/137] #10920 fixing skomos typo

---
 doc/sphinx-guides/source/admin/metadatacustomization.rst        | 2 +-
 .../java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index 8752f11c1e5..e5326efebef 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -579,7 +579,7 @@ In general, the external vocabulary support mechanism may be a better choice for
 The specifics of the user interface for entering/selecting a vocabulary term and how that term is then displayed are managed by third-party Javascripts. The initial Javascripts that have been created provide auto-completion, displaying a list of choices that match what the user has typed so far, but other interfaces, such as displaying a tree of options for a hierarchical vocabulary, are possible. 
 Similarly, existing scripts do relatively simple things for displaying a term - showing the term's name in the appropriate language and providing a link to an external URL with more information, but more sophisticated displays are possible.
 
-Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org), retrieving ORCIDs (from https://orcid.org), services based on Ontoportal product (see https://ontoportal.org/), and using ROR (https://ror.org/) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.)
+Scripts supporting use of vocabularies from services supporting the SKOSMOS protocol (see https://skosmos.org), retrieving ORCIDs (from https://orcid.org), services based on Ontoportal product (see https://ontoportal.org/), and using ROR (https://ror.org/) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.)
 
 Configuration involves specifying which fields are to be mapped, to which Solr field they should be indexed, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters, including the ability to send HTTP headers on calls to the service.
 These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing.
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
index ff78b0c83ec..91150b79505 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
@@ -424,7 +424,7 @@ public Set<String> getIndexableStringsByTermUri(String termUri, JsonObject cvocE
                             for (int i = 0; i < jarr.size(); i++) {
                                 if (jarr.get(i).getValueType().equals(JsonValue.ValueType.STRING)) {
                                     strings.add(jarr.getString(i));
-                                } else if (jarr.get(i).getValueType().equals(ValueType.OBJECT)) { // This condition handles SKOMOS format like [{"lang": "en","value": "non-apis bee"},{"lang": "fr","value": "abeille non apis"}]
+                                } else if (jarr.get(i).getValueType().equals(ValueType.OBJECT)) { // This condition handles SKOSMOS format like [{"lang": "en","value": "non-apis bee"},{"lang": "fr","value": "abeille non apis"}]
                                     JsonObject entry = jarr.getJsonObject(i);
                                     if (entry.containsKey("value")) {
                                         logger.fine("adding " + entry.getString("value") + " for " + termUri);

From a242d14c75db789044792b5f5649de6aeed541af Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Mon, 14 Oct 2024 10:56:19 +0200
Subject: [PATCH 066/137] mime type of m4a uploaded in zip

---
 .../propertyFiles/MimeTypeDetectionByFileExtension.properties    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
index 630539d912e..05e61a40c17 100644
--- a/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
+++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
@@ -15,6 +15,7 @@ m=text/x-matlab
 mat=application/matlab-mat
 md=text/markdown
 mp3=audio/mp3
+m4a=audio/x-m4a
 nii=image/nii
 nc=application/netcdf
 ods=application/vnd.oasis.opendocument.spreadsheet

From cce22a281465959d776cd64c759b728a19cb3721 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Oct 2024 11:54:53 +0100
Subject: [PATCH 067/137] Changed: users/token GET endpoint to support all
 available auth mechanisms

---
 .../edu/harvard/iq/dataverse/api/Users.java   | 21 ++++++++++++-------
 .../edu/harvard/iq/dataverse/api/UsersIT.java |  3 +--
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java
index c1a7c95dbff..ecf7839e616 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java
@@ -137,15 +137,20 @@ public Response deleteToken(@Context ContainerRequestContext crc) {
     @Path("token")
     @AuthRequired
     @GET
-    public Response getTokenExpirationDate() {
-        ApiToken token = authSvc.findApiToken(getRequestApiKey());
-        
-        if (token == null) {
-            return notFound("Token " + getRequestApiKey() + " not found.");
+    public Response getTokenExpirationDate(@Context ContainerRequestContext crc) {
+        try {
+            AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+            ApiToken token = authSvc.findApiTokenByUser(user);
+
+            if (token == null) {
+                return notFound("Token not found.");
+            }
+
+            return ok(String.format("Token %s expires on %s", token.getTokenString(), token.getExpireTime()));
+
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
         }
-        
-        return ok("Token " + getRequestApiKey() + " expires on " + token.getExpireTime());
-        
     }
     
     @Path("token/recreate")
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java
index 1003c1a990c..ce3b8bf75ff 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java
@@ -405,7 +405,6 @@ public void testAPITokenEndpoints() {
         */
         
         createUser = UtilIT.createRandomUser();
-        String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
         Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
         createDataverseResponse.prettyPrint();
@@ -428,7 +427,7 @@ public void testAPITokenEndpoints() {
         getExpiration = UtilIT.getTokenExpiration(tokenForPrivateUrlUser);
         getExpiration.prettyPrint();
         getExpiration.then().assertThat()
-                .statusCode(NOT_FOUND.getStatusCode());
+                .statusCode(UNAUTHORIZED.getStatusCode());
 
         createUser = UtilIT.createRandomUser();
         assertEquals(OK.getStatusCode(), createUser.getStatusCode());

From 129c80c768a7c1e4fa2ec55dcacade723de24f94 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Oct 2024 12:09:40 +0100
Subject: [PATCH 068/137] Added: release notes for #10914

---
 doc/release-notes/10914-users-token-api-credentials.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/10914-users-token-api-credentials.md

diff --git a/doc/release-notes/10914-users-token-api-credentials.md b/doc/release-notes/10914-users-token-api-credentials.md
new file mode 100644
index 00000000000..888214481f6
--- /dev/null
+++ b/doc/release-notes/10914-users-token-api-credentials.md
@@ -0,0 +1,3 @@
+Extended the users/token GET endpoint to support any auth mechanism for retrieving the token information.
+
+Previously, this endpoint only accepted an API token to retrieve its information. Now, it accepts any authentication mechanism and returns the associated API token information.

From 7f5b0bea1670b5c2ec84651b45820211b9df2988 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 15 Oct 2024 13:34:34 +0100
Subject: [PATCH 069/137] Added: updateDataverse endpoint with addDataverse
 refactoring

---
 .../harvard/iq/dataverse/api/Dataverses.java  | 160 ++++++++++----
 .../command/impl/CreateDataverseCommand.java  |   6 +-
 .../command/impl/UpdateDataverseCommand.java  | 204 ++++++++++--------
 3 files changed, 231 insertions(+), 139 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 0ee146ed99b..b85ee0afc8f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -127,73 +127,145 @@ public Response addRoot(@Context ContainerRequestContext crc, String body) {
     @Path("{identifier}")
     public Response addDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String parentIdtf) {
         Dataverse newDataverse;
-        JsonObject newDataverseJson;
         try {
-            newDataverseJson = JsonUtil.getJsonObject(body);
-            newDataverse = jsonParser().parseDataverse(newDataverseJson);
+            newDataverse = parseAndValidateDataverse(body);
         } catch (JsonParsingException jpe) {
-            logger.log(Level.SEVERE, "Json: {0}", body);
             return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
         } catch (JsonParseException ex) {
-            logger.log(Level.SEVERE, "Error parsing dataverse from json: " + ex.getMessage(), ex);
             return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparsetodataverse"), ex.getMessage()));
         }
 
         try {
-            JsonObject metadataBlocksJson = newDataverseJson.getJsonObject("metadataBlocks");
-            List<DataverseFieldTypeInputLevel> inputLevels = null;
-            List<MetadataBlock> metadataBlocks = null;
-            List<DatasetFieldType> facetList = null;
-            if (metadataBlocksJson != null) {
-                JsonArray inputLevelsArray = metadataBlocksJson.getJsonArray("inputLevels");
-                inputLevels = inputLevelsArray != null ? parseInputLevels(inputLevelsArray, newDataverse) : null;
-
-                JsonArray metadataBlockNamesArray = metadataBlocksJson.getJsonArray("metadataBlockNames");
-                metadataBlocks = metadataBlockNamesArray != null ? parseNewDataverseMetadataBlocks(metadataBlockNamesArray) : null;
-
-                JsonArray facetIdsArray = metadataBlocksJson.getJsonArray("facetIds");
-                facetList = facetIdsArray != null ? parseFacets(facetIdsArray) : null;
-            }
+            List<DataverseFieldTypeInputLevel> inputLevels = parseInputLevels(body, newDataverse);
+            List<MetadataBlock> metadataBlocks = parseMetadataBlocks(body);
+            List<DatasetFieldType> facets = parseFacets(body);
 
             if (!parentIdtf.isEmpty()) {
                 Dataverse owner = findDataverseOrDie(parentIdtf);
                 newDataverse.setOwner(owner);
             }
 
-            // set the dataverse - contact relationship in the contacts
-            for (DataverseContact dc : newDataverse.getDataverseContacts()) {
-                dc.setDataverse(newDataverse);
-            }
-
             AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
-            newDataverse = execCommand(new CreateDataverseCommand(newDataverse, createDataverseRequest(u), facetList, inputLevels, metadataBlocks));
+            newDataverse = execCommand(new CreateDataverseCommand(newDataverse, createDataverseRequest(u), facets, inputLevels, metadataBlocks));
             return created("/dataverses/" + newDataverse.getAlias(), json(newDataverse));
-        } catch (WrappedResponse ww) {
-
-            String error = ConstraintViolationUtil.getErrorStringForConstraintViolations(ww.getCause());
-            if (!error.isEmpty()) {
-                logger.log(Level.INFO, error);
-                return ww.refineResponse(error);
-            }
-            return ww.getResponse();
 
+        } catch (WrappedResponse ww) {
+            return handleWrappedResponse(ww);
         } catch (EJBException ex) {
-            Throwable cause = ex;
-            StringBuilder sb = new StringBuilder();
-            sb.append("Error creating dataverse.");
-            while (cause.getCause() != null) {
-                cause = cause.getCause();
-                if (cause instanceof ConstraintViolationException) {
-                    sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));
-                }
-            }
-            logger.log(Level.SEVERE, sb.toString());
-            return error(Response.Status.INTERNAL_SERVER_ERROR, "Error creating dataverse: " + sb.toString());
+            return handleEJBException(ex, "Error creating dataverse.");
         } catch (Exception ex) {
             logger.log(Level.SEVERE, "Error creating dataverse", ex);
             return error(Response.Status.INTERNAL_SERVER_ERROR, "Error creating dataverse: " + ex.getMessage());
+        }
+    }
+
+    @PUT
+    @AuthRequired
+    @Path("{identifier}")
+    public Response updateDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String identifier) {
+        Dataverse originalDataverse;
+        try {
+            originalDataverse = findDataverseOrDie(identifier);
+        } catch (WrappedResponse e) {
+            return e.getResponse();
+        }
+
+        Dataverse updatedDataverse;
+        try {
+            updatedDataverse = parseAndValidateDataverse(body);
+        } catch (JsonParsingException jpe) {
+            return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
+        } catch (JsonParseException ex) {
+            return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparsetodataverse"), ex.getMessage()));
+        }
+
+        try {
+            List<DataverseFieldTypeInputLevel> inputLevels = parseInputLevels(body, originalDataverse);
+            List<MetadataBlock> metadataBlocks = parseMetadataBlocks(body);
+            List<DatasetFieldType> facets = parseFacets(body);
+
+            updatedDataverse.setId(originalDataverse.getId());
 
+            AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
+            updatedDataverse = execCommand(new UpdateDataverseCommand(updatedDataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks));
+            return created("/dataverses/" + updatedDataverse.getAlias(), json(updatedDataverse));
+
+        } catch (WrappedResponse ww) {
+            return handleWrappedResponse(ww);
+        } catch (EJBException ex) {
+            return handleEJBException(ex, "Error updating dataverse.");
+        } catch (Exception ex) {
+            logger.log(Level.SEVERE, "Error updating dataverse", ex);
+            return error(Response.Status.INTERNAL_SERVER_ERROR, "Error updating dataverse: " + ex.getMessage());
+        }
+    }
+
+    private Dataverse parseAndValidateDataverse(String body) throws JsonParsingException, JsonParseException {
+        try {
+            JsonObject dataverseJson = JsonUtil.getJsonObject(body);
+            return jsonParser().parseDataverse(dataverseJson);
+        } catch (JsonParsingException jpe) {
+            logger.log(Level.SEVERE, "Json: {0}", body);
+            throw jpe;
+        } catch (JsonParseException ex) {
+            logger.log(Level.SEVERE, "Error parsing dataverse from json: " + ex.getMessage(), ex);
+            throw ex;
+        }
+    }
+
+    private List<DataverseFieldTypeInputLevel> parseInputLevels(String body, Dataverse dataverse) throws WrappedResponse {
+        JsonObject metadataBlocksJson = getMetadataBlocksJson(body);
+        if (metadataBlocksJson == null) {
+            return null;
+        }
+        JsonArray inputLevelsArray = metadataBlocksJson.getJsonArray("inputLevels");
+        return inputLevelsArray != null ? parseInputLevels(inputLevelsArray, dataverse) : null;
+    }
+
+    private List<MetadataBlock> parseMetadataBlocks(String body) throws WrappedResponse {
+        JsonObject metadataBlocksJson = getMetadataBlocksJson(body);
+        if (metadataBlocksJson == null) {
+            return null;
+        }
+        JsonArray metadataBlocksArray = metadataBlocksJson.getJsonArray("metadataBlockNames");
+        return metadataBlocksArray != null ? parseNewDataverseMetadataBlocks(metadataBlocksArray) : null;
+    }
+
+    private List<DatasetFieldType> parseFacets(String body) throws WrappedResponse {
+        JsonObject metadataBlocksJson = getMetadataBlocksJson(body);
+        if (metadataBlocksJson == null) {
+            return null;
+        }
+        JsonArray facetsArray = metadataBlocksJson.getJsonArray("facetIds");
+        return facetsArray != null ? parseFacets(facetsArray) : null;
+    }
+
+    private JsonObject getMetadataBlocksJson(String body) {
+        JsonObject dataverseJson = JsonUtil.getJsonObject(body);
+        return dataverseJson.getJsonObject("metadataBlocks");
+    }
+
+    private Response handleWrappedResponse(WrappedResponse ww) {
+        String error = ConstraintViolationUtil.getErrorStringForConstraintViolations(ww.getCause());
+        if (!error.isEmpty()) {
+            logger.log(Level.INFO, error);
+            return ww.refineResponse(error);
+        }
+        return ww.getResponse();
+    }
+
+    private Response handleEJBException(EJBException ex, String action) {
+        Throwable cause = ex;
+        StringBuilder sb = new StringBuilder();
+        sb.append(action);
+        while (cause.getCause() != null) {
+            cause = cause.getCause();
+            if (cause instanceof ConstraintViolationException) {
+                sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));
+            }
         }
+        logger.log(Level.SEVERE, sb.toString());
+        return error(Response.Status.INTERNAL_SERVER_ERROR, sb.toString());
     }
 
     private List<MetadataBlock> parseNewDataverseMetadataBlocks(JsonArray metadataBlockNamesArray) throws WrappedResponse {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
index 489b36e7cef..2ce16a86297 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
@@ -67,7 +67,6 @@ public CreateDataverseCommand(Dataverse created,
 
     @Override
     public Dataverse execute(CommandContext ctxt) throws CommandException {
-
         Dataverse owner = created.getOwner();
         if (owner == null) {
             if (ctxt.dataverses().isRootDataverseExists()) {
@@ -75,6 +74,10 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
             }
         }
 
+        for (DataverseContact dc : created.getDataverseContacts()) {
+            dc.setDataverse(created);
+        }
+
         if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
             created.setMetadataBlockRoot(true);
             created.setMetadataBlocks(metadataBlocks);
@@ -194,5 +197,4 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
     public boolean onSuccess(CommandContext ctxt, Object r) {
         return ctxt.dataverses().index((Dataverse) r);
     }
-
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index bdb69dc918f..b1670a264bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -1,13 +1,11 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetFieldType;
-import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.Dataverse.DataverseType;
-import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel;
 import edu.harvard.iq.dataverse.authorization.Permission;
 
 import static edu.harvard.iq.dataverse.dataverse.DataverseUtil.validateDataverseMetadataExternally;
+
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
@@ -21,121 +19,141 @@
 
 /**
  * Update an existing dataverse.
+ *
  * @author michael
  */
-@RequiredPermissions( Permission.EditDataverse )
+@RequiredPermissions(Permission.EditDataverse)
 public class UpdateDataverseCommand extends AbstractCommand<Dataverse> {
-        private static final Logger logger = Logger.getLogger(UpdateDataverseCommand.class.getName());
-	
-	private final Dataverse editedDv;
-	private final List<DatasetFieldType> facetList;
+    private static final Logger logger = Logger.getLogger(UpdateDataverseCommand.class.getName());
+
+    private final Dataverse editedDv;
+    private final List<DatasetFieldType> facetList;
     private final List<Dataverse> featuredDataverseList;
     private final List<DataverseFieldTypeInputLevel> inputLevelList;
+    private final List<MetadataBlock> metadataBlocks;
 
     private boolean datasetsReindexRequired = false;
 
-	public UpdateDataverseCommand(Dataverse editedDv, List<DatasetFieldType> facetList, List<Dataverse> featuredDataverseList, 
-                    DataverseRequest aRequest,  List<DataverseFieldTypeInputLevel> inputLevelList ) {
-            super(aRequest, editedDv);
-            this.editedDv = editedDv;
-            // add update template uses this command but does not
-            // update facet list or featured dataverses
-            if (facetList != null){
-               this.facetList = new ArrayList<>(facetList); 
-            } else {
-               this.facetList = null;
-            }
-            if (featuredDataverseList != null){
-                this.featuredDataverseList = new ArrayList<>(featuredDataverseList);
-            } else {
-                this.featuredDataverseList = null;
-            }
-            if (inputLevelList != null){
-               this.inputLevelList = new ArrayList<>(inputLevelList); 
-            } else {
-               this.inputLevelList = null;
-            }
-	}
-	
-	@Override
-	public Dataverse execute(CommandContext ctxt) throws CommandException {
-            logger.fine("Entering update dataverse command");
-            
-            // Perform any optional validation steps, if defined:
-            if (ctxt.systemConfig().isExternalDataverseValidationEnabled()) {
-                // For admins, an override of the external validation step may be enabled: 
-                if (!(getUser().isSuperuser() && ctxt.systemConfig().isExternalValidationAdminOverrideEnabled())) {
-                    String executable = ctxt.systemConfig().getDataverseValidationExecutable();
-                    boolean result = validateDataverseMetadataExternally(editedDv, executable, getRequest());
-
-                    if (!result) {
-                        String rejectionMessage = ctxt.systemConfig().getDataverseUpdateValidationFailureMsg();
-                        throw new IllegalCommandException(rejectionMessage, this);
-                    }
+    public UpdateDataverseCommand(Dataverse editedDv,
+                                  List<DatasetFieldType> facetList,
+                                  List<Dataverse> featuredDataverseList,
+                                  DataverseRequest aRequest,
+                                  List<DataverseFieldTypeInputLevel> inputLevelList) {
+        this(editedDv, facetList, featuredDataverseList, aRequest, inputLevelList, null);
+    }
+
+    public UpdateDataverseCommand(Dataverse editedDv,
+                                  List<DatasetFieldType> facetList,
+                                  List<Dataverse> featuredDataverseList,
+                                  DataverseRequest aRequest,
+                                  List<DataverseFieldTypeInputLevel> inputLevelList,
+                                  List<MetadataBlock> metadataBlocks) {
+        super(aRequest, editedDv);
+        this.editedDv = editedDv;
+        // add update template uses this command but does not
+        // update facet list or featured dataverses
+        if (facetList != null) {
+            this.facetList = new ArrayList<>(facetList);
+        } else {
+            this.facetList = null;
+        }
+        if (featuredDataverseList != null) {
+            this.featuredDataverseList = new ArrayList<>(featuredDataverseList);
+        } else {
+            this.featuredDataverseList = null;
+        }
+        if (inputLevelList != null) {
+            this.inputLevelList = new ArrayList<>(inputLevelList);
+        } else {
+            this.inputLevelList = null;
+        }
+        if (metadataBlocks != null) {
+            this.metadataBlocks = new ArrayList<>(metadataBlocks);
+        } else {
+            this.metadataBlocks = null;
+        }
+    }
+
+    @Override
+    public Dataverse execute(CommandContext ctxt) throws CommandException {
+        logger.fine("Entering update dataverse command");
+
+        // Perform any optional validation steps, if defined:
+        if (ctxt.systemConfig().isExternalDataverseValidationEnabled()) {
+            // For admins, an override of the external validation step may be enabled:
+            if (!(getUser().isSuperuser() && ctxt.systemConfig().isExternalValidationAdminOverrideEnabled())) {
+                String executable = ctxt.systemConfig().getDataverseValidationExecutable();
+                boolean result = validateDataverseMetadataExternally(editedDv, executable, getRequest());
+
+                if (!result) {
+                    String rejectionMessage = ctxt.systemConfig().getDataverseUpdateValidationFailureMsg();
+                    throw new IllegalCommandException(rejectionMessage, this);
                 }
             }
-            
-            Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
-            
-            DataverseType oldDvType = oldDv.getDataverseType();
-            String oldDvAlias = oldDv.getAlias();
-            String oldDvName = oldDv.getName();
-            oldDv = null; 
-            
-            Dataverse result = ctxt.dataverses().save(editedDv);
-            
-            if ( facetList != null ) {
-                ctxt.facets().deleteFacetsFor(result);
-                int i=0;
-                for ( DatasetFieldType df : facetList ) {
-                    ctxt.facets().create(i++, df.getId(), result.getId());
-                }
+        }
+
+        for (DataverseContact dc : editedDv.getDataverseContacts()) {
+            dc.setDataverse(editedDv);
+        }
+
+        Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
+
+        DataverseType oldDvType = oldDv.getDataverseType();
+        String oldDvAlias = oldDv.getAlias();
+        String oldDvName = oldDv.getName();
+
+        Dataverse result = ctxt.dataverses().save(editedDv);
+
+        if (facetList != null) {
+            ctxt.facets().deleteFacetsFor(result);
+            int i = 0;
+            for (DatasetFieldType df : facetList) {
+                ctxt.facets().create(i++, df.getId(), result.getId());
             }
-            if ( featuredDataverseList != null ) {
-                ctxt.featuredDataverses().deleteFeaturedDataversesFor(result);
-                int i=0;
-                for ( Object obj : featuredDataverseList ) {
-                    Dataverse dv = (Dataverse) obj;
-                    ctxt.featuredDataverses().create(i++, dv.getId(), result.getId());
-                }
+        }
+        if (featuredDataverseList != null) {
+            ctxt.featuredDataverses().deleteFeaturedDataversesFor(result);
+            int i = 0;
+            for (Object obj : featuredDataverseList) {
+                Dataverse dv = (Dataverse) obj;
+                ctxt.featuredDataverses().create(i++, dv.getId(), result.getId());
             }
-            if ( inputLevelList != null ) {
-                ctxt.fieldTypeInputLevels().deleteFacetsFor(result);
-                for ( DataverseFieldTypeInputLevel obj : inputLevelList ) {               
-                    ctxt.fieldTypeInputLevels().create(obj);
-                }
+        }
+        if (inputLevelList != null) {
+            ctxt.fieldTypeInputLevels().deleteFacetsFor(result);
+            for (DataverseFieldTypeInputLevel obj : inputLevelList) {
+                ctxt.fieldTypeInputLevels().create(obj);
             }
-            
-            // We don't want to reindex the children datasets unnecessarily: 
-            // When these values are changed we need to reindex all children datasets
-            // This check is not recursive as all the values just report the immediate parent
-            if (!oldDvType.equals(editedDv.getDataverseType())
+        }
+
+        // We don't want to reindex the children datasets unnecessarily:
+        // When these values are changed we need to reindex all children datasets
+        // This check is not recursive as all the values just report the immediate parent
+        if (!oldDvType.equals(editedDv.getDataverseType())
                 || !oldDvName.equals(editedDv.getName())
                 || !oldDvAlias.equals(editedDv.getAlias())) {
-                datasetsReindexRequired = true;
-            }
-            
-            return result;
-	}
-        
+            datasetsReindexRequired = true;
+        }
+
+        return result;
+    }
+
     @Override
     public boolean onSuccess(CommandContext ctxt, Object r) {
-        
+
         // first kick of async index of datasets
         // TODO: is this actually needed? Is there a better way to handle
         // It appears that we at some point lost some extra logic here, where
         // we only reindex the underlying datasets if one or more of the specific set
-        // of fields have been changed (since these values are included in the 
+        // of fields have been changed (since these values are included in the
         // indexed solr documents for dataasets). So I'm putting that back. -L.A.
         Dataverse result = (Dataverse) r;
-        
+
         if (datasetsReindexRequired) {
             List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
             ctxt.index().asyncIndexDatasetList(datasets, true);
         }
-        
-        return ctxt.dataverses().index((Dataverse) r);
-    }  
 
+        return ctxt.dataverses().index((Dataverse) r);
+    }
 }
-

From 73dd0dd34064cca50304586588eef749053b9637 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 15 Oct 2024 17:11:43 -0400
Subject: [PATCH 070/137] fix relationType display value bug

---
 src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
index 0433c425fd2..ac5923b95bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
@@ -1390,7 +1390,10 @@ public List<DatasetRelPublication> getRelatedPublications() {
                                 relatedPublication.setIdNumber(subField.getDisplayValue());
                                 break;
                             case DatasetFieldConstant.publicationRelationType:
-                                relatedPublication.setRelationType(subField.getDisplayValue());
+                                List<String> values = subField.getValues_nondisplay();
+                                if (!values.isEmpty()) {
+                                    relatedPublication.setRelationType(values.get(0)); //only one value allowed
+                                }
                                 break;
                         }
                     }

From d039a108f992f170d5323a26a6d6f7dafb14029b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 16 Oct 2024 08:59:11 -0400
Subject: [PATCH 071/137] IQSS/10697 - Improve batch permission indexing
 (#10698)

* reindex batches of 20 files instead of all at once

* Also only keep 100 files in list at a time

* release note

* Just do collections/datasets as you go

Avoids keeping everything in memory, also helps in tracking progress as
you can see the permissionindextime getting updated per dataset.

* fix merge issues, add logging

* put comments back to how they were #10697

* reduce logging #10697

* rename release note and add PR number #10697

* fix logging - finest for per file, space in message

* adding a space in log message - per review

---------

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 .../10697-improve-permission-indexing.md      |   7 +
 .../search/SolrIndexServiceBean.java          | 151 ++++++++++--------
 2 files changed, 91 insertions(+), 67 deletions(-)
 create mode 100644 doc/release-notes/10697-improve-permission-indexing.md

diff --git a/doc/release-notes/10697-improve-permission-indexing.md b/doc/release-notes/10697-improve-permission-indexing.md
new file mode 100644
index 00000000000..b232b1c4d3c
--- /dev/null
+++ b/doc/release-notes/10697-improve-permission-indexing.md
@@ -0,0 +1,7 @@
+### Reindexing after a role assignment is less memory intensive
+
+Adding/removing a user from a role on a collection, particularly the root collection, could lead to a significant increase in memory use resulting in Dataverse itself failing with an out-of-memory condition. Such changes now consume much less memory.
+
+If you have experienced out-of-memory failures in Dataverse in the past that could have been caused by this problem, you may wish to run a [reindex in place](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#reindex-in-place) to update any out-of-date information.
+
+For more information, see #10697 and #10698.
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java
index cfe29ea08c7..e4d885276d0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java
@@ -34,7 +34,7 @@
 public class SolrIndexServiceBean {
 
     private static final Logger logger = Logger.getLogger(SolrIndexServiceBean.class.getCanonicalName());
-    
+
     @EJB
     DvObjectServiceBean dvObjectService;
     @EJB
@@ -149,7 +149,7 @@ private List<DvObjectSolrDoc> constructDatasetSolrDocs(Dataset dataset) {
         return solrDocs;
     }
 
-//    private List<DvObjectSolrDoc> constructDatafileSolrDocs(DataFile dataFile) {
+    // private List<DvObjectSolrDoc> constructDatafileSolrDocs(DataFile dataFile) {
     private List<DvObjectSolrDoc> constructDatafileSolrDocs(DataFile dataFile, Map<Long, List<String>> permStringByDatasetVersion) {
         List<DvObjectSolrDoc> datafileSolrDocs = new ArrayList<>();
         Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataFile.getOwner());
@@ -166,14 +166,14 @@ private List<DvObjectSolrDoc> constructDatafileSolrDocs(DataFile dataFile, Map<L
                         cachedPerms = permStringByDatasetVersion.get(datasetVersionFileIsAttachedTo.getId());
                     }
                     if (cachedPerms != null) {
-                        logger.fine("reusing cached perms for file " + dataFile.getId());
+                        logger.finest("reusing cached perms for file " + dataFile.getId());
                         perms = cachedPerms;
                     } else if (datasetVersionFileIsAttachedTo.isReleased()) {
-                        logger.fine("no cached perms, file is public/discoverable/searchable for file " + dataFile.getId());
+                        logger.finest("no cached perms, file is public/discoverable/searchable for file " + dataFile.getId());
                         perms.add(IndexServiceBean.getPublicGroupString());
                     } else {
                         // go to the well (slow)
-                        logger.fine("no cached perms, file is not public, finding perms for file " + dataFile.getId());
+                        logger.finest("no cached perms, file is not public, finding perms for file " + dataFile.getId());
                         perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo);
                     }
                 } else {
@@ -204,13 +204,14 @@ private List<DvObjectSolrDoc> constructDatafileSolrDocsFromDataset(Dataset datas
                 } else {
                     perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo);
                 }
+
                 for (FileMetadata fileMetadata : datasetVersionFileIsAttachedTo.getFileMetadatas()) {
                     Long fileId = fileMetadata.getDataFile().getId();
                     String solrIdStart = IndexServiceBean.solrDocIdentifierFile + fileId;
                     String solrIdEnd = getDatasetOrDataFileSolrEnding(datasetVersionFileIsAttachedTo.getVersionState());
                     String solrId = solrIdStart + solrIdEnd;
                     DvObjectSolrDoc dataFileSolrDoc = new DvObjectSolrDoc(fileId.toString(), solrId, datasetVersionFileIsAttachedTo.getId(), fileMetadata.getLabel(), perms);
-                    logger.fine("adding fileid " + fileId);
+                    logger.finest("adding fileid " + fileId);
                     datafileSolrDocs.add(dataFileSolrDoc);
                 }
             }
@@ -361,20 +362,19 @@ private void persistToSolr(Collection<SolrInputDocument> docs) throws SolrServer
 
     public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) {
         DvObject definitionPoint = dvObjectService.findDvObject(definitionPointId);
-        if ( definitionPoint == null ) {
+        if (definitionPoint == null) {
             logger.log(Level.WARNING, "Cannot find a DvOpbject with id of {0}", definitionPointId);
             return null;
         } else {
             return indexPermissionsOnSelfAndChildren(definitionPoint);
         }
     }
-    
+
     /**
      * We use the database to determine direct children since there is no
      * inheritance
      */
     public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) {
-        List<DvObject> dvObjectsToReindexPermissionsFor = new ArrayList<>();
         List<DataFile> filesToReindexAsBatch = new ArrayList<>();
         /**
          * @todo Re-indexing the definition point itself seems to be necessary
@@ -383,27 +383,47 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint)
 
         // We don't create a Solr "primary/content" doc for the root dataverse
         // so don't create a Solr "permission" doc either.
+        int i = 0;
+        int numObjects = 0;
         if (definitionPoint.isInstanceofDataverse()) {
             Dataverse selfDataverse = (Dataverse) definitionPoint;
             if (!selfDataverse.equals(dataverseService.findRootDataverse())) {
-                dvObjectsToReindexPermissionsFor.add(definitionPoint);
+                indexPermissionsForOneDvObject(definitionPoint);
+                numObjects++;
             }
             List<Dataset> directChildDatasetsOfDvDefPoint = datasetService.findByOwnerId(selfDataverse.getId());
             for (Dataset dataset : directChildDatasetsOfDvDefPoint) {
-                dvObjectsToReindexPermissionsFor.add(dataset);
+                indexPermissionsForOneDvObject(dataset);
+                numObjects++;
                 for (DataFile datafile : filesToReIndexPermissionsFor(dataset)) {
                     filesToReindexAsBatch.add(datafile);
+                    i++;
+                    if (i % 100 == 0) {
+                        reindexFilesInBatches(filesToReindexAsBatch);
+                        filesToReindexAsBatch.clear();
+                    }
+                    if (i % 1000 == 0) {
+                        logger.fine("Progress: " +i + " files permissions reindexed");
+                    }
                 }
+                logger.fine("Progress : dataset " + dataset.getId() + " permissions reindexed");
             }
         } else if (definitionPoint.isInstanceofDataset()) {
-            dvObjectsToReindexPermissionsFor.add(definitionPoint);
+            indexPermissionsForOneDvObject(definitionPoint);
+            numObjects++;
             // index files
             Dataset dataset = (Dataset) definitionPoint;
             for (DataFile datafile : filesToReIndexPermissionsFor(dataset)) {
                 filesToReindexAsBatch.add(datafile);
+                i++;
+                if (i % 100 == 0) {
+                    reindexFilesInBatches(filesToReindexAsBatch);
+                    filesToReindexAsBatch.clear();
+                }
             }
         } else {
-            dvObjectsToReindexPermissionsFor.add(definitionPoint);
+            indexPermissionsForOneDvObject(definitionPoint);
+            numObjects++;
         }
 
         /**
@@ -412,64 +432,64 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint)
          * @todo Should update timestamps, probably, even thought these are
          * files, see https://github.com/IQSS/dataverse/issues/2421
          */
-        String response = reindexFilesInBatches(filesToReindexAsBatch);
-
-        for (DvObject dvObject : dvObjectsToReindexPermissionsFor) {
-            /**
-             * @todo do something with this response
-             */
-            IndexResponse indexResponse = indexPermissionsForOneDvObject(dvObject);
-        }
-        
+        reindexFilesInBatches(filesToReindexAsBatch);
+        logger.fine("Reindexed permissions for " + i + " files and " + numObjects + " datasets/collections");
         return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint
-                + ": " + dvObjectsToReindexPermissionsFor.size()
-        );
+                + ": " + numObjects);
     }
 
     private String reindexFilesInBatches(List<DataFile> filesToReindexPermissionsFor) {
         List<SolrInputDocument> docs = new ArrayList<>();
         Map<Long, List<Long>> byParentId = new HashMap<>();
         Map<Long, List<String>> permStringByDatasetVersion = new HashMap<>();
-        for (DataFile file : filesToReindexPermissionsFor) {
-            Dataset dataset = (Dataset) file.getOwner();
-            Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
-            for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataset)) {
-                boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState());
-                if (cardShouldExist) {
-                    List<String> cachedPermission = permStringByDatasetVersion.get(datasetVersionFileIsAttachedTo.getId());
-                    if (cachedPermission == null) {
-                        logger.fine("no cached permission! Looking it up...");
-                        List<DvObjectSolrDoc> fileSolrDocs = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion);
-                        for (DvObjectSolrDoc fileSolrDoc : fileSolrDocs) {
-                            Long datasetVersionId = fileSolrDoc.getDatasetVersionId();
-                            if (datasetVersionId != null) {
-                                permStringByDatasetVersion.put(datasetVersionId, fileSolrDoc.getPermissions());
+        int i = 0;
+        try {
+            for (DataFile file : filesToReindexPermissionsFor) {
+                Dataset dataset = (Dataset) file.getOwner();
+                Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
+                for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataset)) {
+                    boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState());
+                    if (cardShouldExist) {
+                        List<String> cachedPermission = permStringByDatasetVersion.get(datasetVersionFileIsAttachedTo.getId());
+                        if (cachedPermission == null) {
+                            logger.finest("no cached permission! Looking it up...");
+                            List<DvObjectSolrDoc> fileSolrDocs = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion);
+                            for (DvObjectSolrDoc fileSolrDoc : fileSolrDocs) {
+                                Long datasetVersionId = fileSolrDoc.getDatasetVersionId();
+                                if (datasetVersionId != null) {
+                                    permStringByDatasetVersion.put(datasetVersionId, fileSolrDoc.getPermissions());
+                                    SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc);
+                                    docs.add(solrDoc);
+                                    i++;
+                                }
+                            }
+                        } else {
+                            logger.finest("cached permission is " + cachedPermission);
+                            List<DvObjectSolrDoc> fileSolrDocsBasedOnCachedPermissions = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion);
+                            for (DvObjectSolrDoc fileSolrDoc : fileSolrDocsBasedOnCachedPermissions) {
                                 SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc);
                                 docs.add(solrDoc);
+                                i++;
                             }
                         }
-                    } else {
-                        logger.fine("cached permission is " + cachedPermission);
-                        List<DvObjectSolrDoc> fileSolrDocsBasedOnCachedPermissions = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion);
-                        for (DvObjectSolrDoc fileSolrDoc : fileSolrDocsBasedOnCachedPermissions) {
-                            SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc);
-                            docs.add(solrDoc);
+                        if (i % 20 == 0) {
+                            persistToSolr(docs);
+                            docs = new ArrayList<>();
                         }
                     }
                 }
+                Long parent = file.getOwner().getId();
+                List<Long> existingList = byParentId.get(parent);
+                if (existingList == null) {
+                    List<Long> empty = new ArrayList<>();
+                    byParentId.put(parent, empty);
+                } else {
+                    List<Long> updatedList = existingList;
+                    updatedList.add(file.getId());
+                    byParentId.put(parent, updatedList);
+                }
             }
-            Long parent = file.getOwner().getId();
-            List<Long> existingList = byParentId.get(parent);
-            if (existingList == null) {
-                List<Long> empty = new ArrayList<>();
-                byParentId.put(parent, empty);
-            } else {
-                List<Long> updatedList = existingList;
-                updatedList.add(file.getId());
-                byParentId.put(parent, updatedList);
-            }
-        }
-        try {
+
             persistToSolr(docs);
             return " " + filesToReindexPermissionsFor.size() + " files indexed across " + docs.size() + " Solr documents ";
         } catch (SolrServerException | IOException ex) {
@@ -517,29 +537,26 @@ public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServer
     }
 
     /**
-     * 
-     *
      * @return A list of dvobject ids that should have their permissions
-     * re-indexed because Solr was down when a permission was added. The permission
-     * should be added to Solr. The id of the permission contains the type of
-     * DvObject and the primary key of the dvObject.
-     * DvObjects of type DataFile are currently skipped because their index
-     * time isn't stored in the database, since they are indexed along 
-     * with their parent dataset (this may change).
+     * re-indexed because Solr was down when a permission was added. The
+     * permission should be added to Solr. The id of the permission contains the
+     * type of DvObject and the primary key of the dvObject. DvObjects of type
+     * DataFile are currently skipped because their index time isn't stored in
+     * the database, since they are indexed along with their parent dataset
+     * (this may change).
      */
     public List<Long> findPermissionsInDatabaseButStaleInOrMissingFromSolr() {
         List<Long> indexingRequired = new ArrayList<>();
         long rootDvId = dataverseService.findRootDataverse().getId();
         List<Long> missingDataversePermissionIds = dataverseService.findIdStalePermission();
         List<Long> missingDatasetPermissionIds = datasetService.findIdStalePermission();
-        for (Long id : missingDataversePermissionIds) {          
+        for (Long id : missingDataversePermissionIds) {
             if (!id.equals(rootDvId)) {
-            indexingRequired.add(id);
+                indexingRequired.add(id);
             }
         }
         indexingRequired.addAll(missingDatasetPermissionIds);
         return indexingRequired;
     }
 
-  
 }

From 19c8a12b32a502ee43f46916248a7d4691928aa6 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Oct 2024 16:30:00 +0100
Subject: [PATCH 072/137] Changed: limiting the information to update in a
 dataverse through the new update endpoint

---
 .../harvard/iq/dataverse/api/Dataverses.java  |  8 +-
 .../iq/dataverse/util/json/JsonParser.java    | 79 ++++++++++++++-----
 2 files changed, 62 insertions(+), 25 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index b85ee0afc8f..0bc389041c2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -128,7 +128,7 @@ public Response addRoot(@Context ContainerRequestContext crc, String body) {
     public Response addDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String parentIdtf) {
         Dataverse newDataverse;
         try {
-            newDataverse = parseAndValidateDataverse(body);
+            newDataverse = parseAndValidateDataverseRequestBody(body, null);
         } catch (JsonParsingException jpe) {
             return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
         } catch (JsonParseException ex) {
@@ -172,7 +172,7 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
 
         Dataverse updatedDataverse;
         try {
-            updatedDataverse = parseAndValidateDataverse(body);
+            updatedDataverse = parseAndValidateDataverseRequestBody(body, originalDataverse);
         } catch (JsonParsingException jpe) {
             return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
         } catch (JsonParseException ex) {
@@ -200,10 +200,10 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
         }
     }
 
-    private Dataverse parseAndValidateDataverse(String body) throws JsonParsingException, JsonParseException {
+    private Dataverse parseAndValidateDataverseRequestBody(String body, Dataverse dataverseToUpdate) throws JsonParsingException, JsonParseException {
         try {
             JsonObject dataverseJson = JsonUtil.getJsonObject(body);
-            return jsonParser().parseDataverse(dataverseJson);
+            return dataverseToUpdate != null ? jsonParser().parseDataverseUpdates(dataverseJson, dataverseToUpdate) : jsonParser().parseDataverse(dataverseJson);
         } catch (JsonParsingException jpe) {
             logger.log(Level.SEVERE, "Json: {0}", body);
             throw jpe;
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index 2f01c9bc2f2..f63e4c4fd9c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -50,6 +50,7 @@
 import java.util.Set;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
@@ -128,19 +129,8 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
         dv.setPermissionRoot(jobj.getBoolean("permissionRoot", false));
         dv.setFacetRoot(jobj.getBoolean("facetRoot", false));
         dv.setAffiliation(jobj.getString("affiliation", null));
-      
-        if (jobj.containsKey("dataverseContacts")) {
-            JsonArray dvContacts = jobj.getJsonArray("dataverseContacts");
-            int i = 0;
-            List<DataverseContact> dvContactList = new LinkedList<>();
-            for (JsonValue jsv : dvContacts) {
-                DataverseContact dvc = new DataverseContact(dv);
-                dvc.setContactEmail(getMandatoryString((JsonObject) jsv, "contactEmail"));
-                dvc.setDisplayOrder(i++);
-                dvContactList.add(dvc);
-            }
-            dv.setDataverseContacts(dvContactList);
-        }
+
+        updateDataverseContacts(dv, jobj);
         
         if (jobj.containsKey("theme")) {
             DataverseTheme theme = parseDataverseTheme(jobj.getJsonObject("theme"));
@@ -149,14 +139,8 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
         }
 
         dv.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED); // default
-        if (jobj.containsKey("dataverseType")) {
-            for (Dataverse.DataverseType dvtype : Dataverse.DataverseType.values()) {
-                if (dvtype.name().equals(jobj.getString("dataverseType"))) {
-                    dv.setDataverseType(dvtype);
-                }
-            }
-        }
-        
+        updateDataverseType(dv, jobj);
+
         if (jobj.containsKey("filePIDsEnabled")) {
             dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled"));
         }
@@ -189,6 +173,59 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
                 
         return dv;
     }
+
+    public Dataverse parseDataverseUpdates(JsonObject jsonObject, Dataverse dataverseToUpdate) throws JsonParseException {
+        String alias = jsonObject.getString("alias", null);
+        if (alias != null) {
+            dataverseToUpdate.setAlias(alias);
+        }
+
+        String name = jsonObject.getString("name", null);
+        if (name != null) {
+            dataverseToUpdate.setName(name);
+        }
+
+        String description = jsonObject.getString("description", null);
+        if (description != null) {
+            dataverseToUpdate.setDescription(description);
+        }
+
+        String affiliation = jsonObject.getString("affiliation", null);
+        if (affiliation != null) {
+            dataverseToUpdate.setAffiliation(affiliation);
+        }
+
+        updateDataverseType(dataverseToUpdate, jsonObject);
+
+        updateDataverseContacts(dataverseToUpdate, jsonObject);
+
+        return dataverseToUpdate;
+    }
+
+    private void updateDataverseType(Dataverse dataverse, JsonObject jsonObject) {
+        String receivedDataverseType = jsonObject.getString("dataverseType", null);
+        if (receivedDataverseType != null) {
+            Arrays.stream(Dataverse.DataverseType.values())
+                    .filter(type -> type.name().equals(receivedDataverseType))
+                    .findFirst()
+                    .ifPresent(dataverse::setDataverseType);
+        }
+    }
+
+    private void updateDataverseContacts(Dataverse dataverse, JsonObject jsonObject) throws JsonParseException {
+        if (jsonObject.containsKey("dataverseContacts")) {
+            JsonArray dvContacts = jsonObject.getJsonArray("dataverseContacts");
+            int i = 0;
+            List<DataverseContact> dvContactList = new LinkedList<>();
+            for (JsonValue jsv : dvContacts) {
+                DataverseContact dvc = new DataverseContact(dataverse);
+                dvc.setContactEmail(getMandatoryString((JsonObject) jsv, "contactEmail"));
+                dvc.setDisplayOrder(i++);
+                dvContactList.add(dvc);
+            }
+            dataverse.setDataverseContacts(dvContactList);
+        }
+    }
     
     public DataverseTheme parseDataverseTheme(JsonObject obj) {
 

From f4c3d2c9d9991edd2d02bd760d1fec86547a519c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Oct 2024 16:43:12 +0100
Subject: [PATCH 073/137] Removed: DataverseContact host dataverse re-set

---
 .../dataverse/engine/command/impl/CreateDataverseCommand.java | 4 ----
 .../dataverse/engine/command/impl/UpdateDataverseCommand.java | 4 ----
 2 files changed, 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
index 2ce16a86297..6957dac416d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
@@ -74,10 +74,6 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
             }
         }
 
-        for (DataverseContact dc : created.getDataverseContacts()) {
-            dc.setDataverse(created);
-        }
-
         if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
             created.setMetadataBlockRoot(true);
             created.setMetadataBlocks(metadataBlocks);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index b1670a264bf..551f0ffdff7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -92,10 +92,6 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
             }
         }
 
-        for (DataverseContact dc : editedDv.getDataverseContacts()) {
-            dc.setDataverse(editedDv);
-        }
-
         Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
 
         DataverseType oldDvType = oldDv.getDataverseType();

From 330462e16a7b231cb5894fbb0f04fd8fd6cb2701 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 16 Oct 2024 17:01:15 -0400
Subject: [PATCH 074/137] Additional DataCiteXML testing with more fields

---
 .../doi/datacite/XmlMetadataTemplateTest.java | 289 +++++++++++++++++-
 1 file changed, 287 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java
index c03146904de..f282e681175 100644
--- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java
@@ -1,15 +1,21 @@
 package edu.harvard.iq.dataverse.pidproviders.doi.datacite;
 
+import edu.harvard.iq.dataverse.ControlledVocabularyValue;
+import edu.harvard.iq.dataverse.DataCitation;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetAuthor;
 import edu.harvard.iq.dataverse.DatasetField;
+import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
 import edu.harvard.iq.dataverse.DatasetFieldConstant;
 import edu.harvard.iq.dataverse.DatasetFieldType;
+import edu.harvard.iq.dataverse.DatasetFieldValue;
 import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
+import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.MetadataBlock;
 import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.dataset.DatasetType;
@@ -20,16 +26,30 @@
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.json.CompoundVocabularyException;
+import edu.harvard.iq.dataverse.util.json.ControlledVocabularyException;
+import edu.harvard.iq.dataverse.util.json.JsonParseException;
+import edu.harvard.iq.dataverse.util.json.JsonParser;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
 import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import edu.harvard.iq.dataverse.util.xml.XmlValidator;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonObject;
+import jakarta.json.JsonString;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import javax.xml.transform.stream.StreamSource;
 
@@ -72,7 +92,7 @@ public static void setupMocks() {
 
     }
 
-    /**
+    /** A minimal example to assure that the XMLMetadataTemplate generates output consistent with the DataCite XML v4.5 schema.
      */
     @Test
     public void testDataCiteXMLCreation() throws IOException {
@@ -135,7 +155,61 @@ public void testDataCiteXMLCreation() throws IOException {
         d.setDatasetType(dType);
 
         String xml = template.generateXML(d);
-        System.out.println("Output is " + xml);
+        System.out.println("Output from minimal example is " + xml);
+        try {
+            StreamSource source = new StreamSource(new StringReader(xml));
+            source.setSystemId("DataCite XML for test dataset");
+            assertTrue(XmlValidator.validateXmlSchema(source, new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd")));
+        } catch (SAXException e) {
+            System.out.println("Invalid schema: " + e.getMessage());
+        }
+        
+    }
+
+    /**
+     * This tests a more complete example based off of the dataset-all-defaults
+     * file, again checking for conformance of the result with the DataCite XML v4.5
+     * schema.
+     */
+    @Test
+    public void testDataCiteXMLCreationAllFields() throws IOException {
+        Dataverse collection = new Dataverse();
+        collection.setCitationDatasetFieldTypes(new ArrayList<>());
+        Dataset d = new Dataset();
+        d.setOwner(collection);
+        DatasetVersion dv = new DatasetVersion();
+        TermsOfUseAndAccess toa = new TermsOfUseAndAccess();
+        toa.setTermsOfUse("Some terms");
+        dv.setTermsOfUseAndAccess(toa);
+        dv.setDataset(d);
+        DatasetFieldType primitiveDSFType = new DatasetFieldType(DatasetFieldConstant.title,
+                DatasetFieldType.FieldType.TEXT, false);
+        DatasetField testDatasetField = new DatasetField();
+
+        dv.setVersionState(VersionState.DRAFT);
+
+        testDatasetField.setDatasetVersion(dv);
+        
+        File datasetVersionJson = new File("src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt");
+        String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath())));
+        JsonObject datasetJson = JsonUtil.getJsonObject(datasetVersionAsJson);
+        
+        GlobalId doi = new GlobalId("doi", datasetJson.getString("authority"), datasetJson.getString("identifier"), null, null, null);
+        d.setGlobalId(doi);
+
+        
+        List<DatasetField> fields = assertDoesNotThrow(() -> XmlMetadataTemplateTest.parseMetadataBlocks(datasetJson.getJsonObject("datasetVersion").getJsonObject("metadataBlocks")));
+        dv.setDatasetFields(fields);
+        
+        ArrayList<DatasetVersion> dsvs = new ArrayList<>();
+        dsvs.add(0, dv);
+        d.setVersions(dsvs);
+        DatasetType dType = new DatasetType();
+        dType.setName(DatasetType.DATASET_TYPE_DATASET);
+        d.setDatasetType(dType);
+        String xml = DOIDataCiteRegisterService.getMetadataFromDvObject(
+                dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset());
+        System.out.println("Output from dataset-all-defaults is " + xml);
         try {
             StreamSource source = new StreamSource(new StringReader(xml));
             source.setSystemId("DataCite XML for test dataset");
@@ -146,4 +220,215 @@ public void testDataCiteXMLCreation() throws IOException {
         
     }
 
+    
+    /**
+     * Mock Utility Methods - These methods support importing DatasetFields from the
+     * Dataverse JSON export format. They assume that any DatasetFieldType
+     * referenced exists, that any Controlled Vocabulary value exists, etc. which
+     * avoids having to do database lookups or read metadatablock tsv files. They
+     * are derived from the JsonParser methods of the same names with any db
+     * references and DatasetFieldType-related error checking removed.
+     */
+    public static List<DatasetField> parseMetadataBlocks(JsonObject json) throws JsonParseException {
+        
+        Map<String, DatasetFieldType> existingTypes = new HashMap<>();
+        
+        Set<String> keys = json.keySet();
+        List<DatasetField> fields = new LinkedList<>();
+
+        for (String blockName : keys) {
+            MetadataBlock block = new MetadataBlock();
+            block.setName(blockName);
+            JsonObject blockJson = json.getJsonObject(blockName);
+            JsonArray fieldsJson = blockJson.getJsonArray("fields");
+            fields.addAll(parseFieldsFromArray(fieldsJson, true, block, existingTypes));
+        }
+        return fields;
+    }
+
+    private static List<DatasetField> parseFieldsFromArray(JsonArray fieldsArray, Boolean testType, MetadataBlock block,
+            Map<String, DatasetFieldType> existingTypes) throws JsonParseException {
+        List<DatasetField> fields = new LinkedList<>();
+        for (JsonObject fieldJson : fieldsArray.getValuesAs(JsonObject.class)) {
+
+            DatasetField field = parseField(fieldJson, testType, block, existingTypes);
+            if (field != null) {
+                fields.add(field);
+            }
+
+        }
+        return fields;
+
+    }    
+
+    
+    public  static DatasetField parseField(JsonObject json, Boolean testType,  MetadataBlock block, Map<String, DatasetFieldType> existingTypes) throws JsonParseException {
+        if (json == null) {
+            return null;
+        }
+
+        DatasetField ret = new DatasetField();
+        String fieldName = json.getString("typeName", "");
+        String typeClass = json.getString("typeClass", "");
+        if(!existingTypes.containsKey(fieldName)) {
+            boolean multiple = json.getBoolean("multiple");
+            DatasetFieldType fieldType = new DatasetFieldType();
+            fieldType.setName(fieldName);
+            fieldType.setAllowMultiples(multiple);
+            fieldType.setAllowControlledVocabulary(typeClass.equals("controlledVocabulary"));
+            fieldType.setFieldType(FieldType.TEXT);
+            fieldType.setMetadataBlock(block);
+            fieldType.setChildDatasetFieldTypes(new ArrayList<>());
+            existingTypes.put(fieldName, fieldType);
+        }
+        DatasetFieldType type = existingTypes.get(fieldName);
+        ret.setDatasetFieldType(type);
+
+        if (typeClass.equals("compound")) {
+            parseCompoundValue(ret, type, json, testType, block, existingTypes);
+        } else if (type.isControlledVocabulary()) {
+            parseControlledVocabularyValue(ret, type, json);
+        } else {
+            parsePrimitiveValue(ret, type, json);
+        }
+
+        return ret;
+    }
+    
+    public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json, Boolean testType, MetadataBlock block, Map<String, DatasetFieldType> existingTypes) throws JsonParseException {
+        List<ControlledVocabularyException> vocabExceptions = new ArrayList<>();
+        List<DatasetFieldCompoundValue> vals = new LinkedList<>();
+        if (compoundType.isAllowMultiples()) {
+            int order = 0;
+            try {
+                json.getJsonArray("value").getValuesAs(JsonObject.class);
+            } catch (ClassCastException cce) {
+                throw new JsonParseException("Invalid values submitted for " + compoundType.getName() + ". It should be an array of values.");
+            }
+            for (JsonObject obj : json.getJsonArray("value").getValuesAs(JsonObject.class)) {
+                DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue();
+                List<DatasetField> fields = new LinkedList<>();
+                for (String fieldName : obj.keySet()) {
+                    JsonObject childFieldJson = obj.getJsonObject(fieldName);
+                    DatasetField f=null;
+                    try {
+                        f = parseField(childFieldJson, testType, block, existingTypes);
+                    } catch(ControlledVocabularyException ex) {
+                        vocabExceptions.add(ex);
+                    }
+                    
+                    if (f!=null) {
+                        f.setParentDatasetFieldCompoundValue(cv);
+                            fields.add(f);
+                    }
+                }
+                if (!fields.isEmpty()) {
+                    cv.setChildDatasetFields(fields);
+                    cv.setDisplayOrder(order);
+                    vals.add(cv);
+                }
+                order++;
+            }
+
+           
+
+        } else {
+            
+            DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue();
+            List<DatasetField> fields = new LinkedList<>();
+            JsonObject value = json.getJsonObject("value");
+            for (String key : value.keySet()) {
+                JsonObject childFieldJson = value.getJsonObject(key);
+                DatasetField f = null;
+                try {
+                    f=parseField(childFieldJson, testType, block, existingTypes);
+                } catch(ControlledVocabularyException ex ) {
+                    vocabExceptions.add(ex);
+                }
+                if (f!=null) {
+                    f.setParentDatasetFieldCompoundValue(cv);
+                    fields.add(f);
+                }
+            }
+            if (!fields.isEmpty()) {
+                cv.setChildDatasetFields(fields);
+                vals.add(cv);
+            }
+      
+    }
+        if (!vocabExceptions.isEmpty()) {
+            throw new CompoundVocabularyException( "Invalid controlled vocabulary in compound field ", vocabExceptions, vals);
+        }
+
+        for (DatasetFieldCompoundValue dsfcv : vals) {
+            dsfcv.setParentDatasetField(dsf);
+        }
+        dsf.setDatasetFieldCompoundValues(vals);
+    }
+
+    public static void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObject json) throws JsonParseException {
+        List<DatasetFieldValue> vals = new LinkedList<>();
+        if (dft.isAllowMultiples()) {
+           try {
+            json.getJsonArray("value").getValuesAs(JsonObject.class);
+            } catch (ClassCastException cce) {
+                throw new JsonParseException("Invalid values submitted for " + dft.getName() + ". It should be an array of values.");
+            }
+            for (JsonString val : json.getJsonArray("value").getValuesAs(JsonString.class)) {
+                DatasetFieldValue datasetFieldValue = new DatasetFieldValue(dsf);
+                datasetFieldValue.setDisplayOrder(vals.size() - 1);
+                datasetFieldValue.setValue(val.getString().trim());
+                vals.add(datasetFieldValue);
+            }
+
+        } else {
+            try {json.getString("value");}
+            catch (ClassCastException cce) {
+                throw new JsonParseException("Invalid value submitted for " + dft.getName() + ". It should be a single value.");
+            }            
+            DatasetFieldValue datasetFieldValue = new DatasetFieldValue();
+            datasetFieldValue.setValue(json.getString("value", "").trim());
+            datasetFieldValue.setDatasetField(dsf);
+            vals.add(datasetFieldValue);
+        }
+
+        dsf.setDatasetFieldValues(vals);
+    }
+    
+    public static void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cvvType, JsonObject json) throws JsonParseException {
+        List<ControlledVocabularyValue> vals = new LinkedList<>();
+        try {
+            if (cvvType.isAllowMultiples()) {
+                try {
+                    json.getJsonArray("value").getValuesAs(JsonObject.class);
+                } catch (ClassCastException cce) {
+                    throw new JsonParseException("Invalid values submitted for " + cvvType.getName() + ". It should be an array of values.");
+                }
+                for (JsonString strVal : json.getJsonArray("value").getValuesAs(JsonString.class)) {
+                    String strValue = strVal.getString();
+                    ControlledVocabularyValue cvv = new ControlledVocabularyValue();
+                    cvv.setDatasetFieldType(cvvType);
+                    cvv.setStrValue(strVal.getString());
+                    vals.add(cvv);
+                }
+
+            } else {
+                try {
+                    json.getString("value");
+                } catch (ClassCastException cce) {
+                    throw new JsonParseException("Invalid value submitted for " + cvvType.getName() + ". It should be a single value.");
+                }
+                String strValue = json.getString("value", "");
+                ControlledVocabularyValue cvv = new ControlledVocabularyValue();
+                cvv.setDatasetFieldType(cvvType);
+                cvv.setStrValue(strValue);
+                vals.add(cvv);
+            }
+        } catch (ClassCastException cce) {
+            throw new JsonParseException("Invalid values submitted for " + cvvType.getName());
+        }
+
+        dsf.setControlledVocabularyValues(vals);
+    }
+
 }

From 816b7047e4f6209c112c6f2bba3c9b22d05a1ca4 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 16 Oct 2024 17:08:13 -0400
Subject: [PATCH 075/137] formatting after review dog notice

---
 .../doi/datacite/XmlMetadataTemplateTest.java | 113 ++++++++++--------
 1 file changed, 63 insertions(+), 50 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java
index f282e681175..2bd6818821d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java
@@ -92,7 +92,9 @@ public static void setupMocks() {
 
     }
 
-    /** A minimal example to assure that the XMLMetadataTemplate generates output consistent with the DataCite XML v4.5 schema.
+    /**
+     * A minimal example to assure that the XMLMetadataTemplate generates output
+     * consistent with the DataCite XML v4.5 schema.
      */
     @Test
     public void testDataCiteXMLCreation() throws IOException {
@@ -126,7 +128,7 @@ public void testDataCiteXMLCreation() throws IOException {
         doiMetadata.setAuthors(authors);
         doiMetadata.setPublisher("Dataverse");
         XmlMetadataTemplate template = new XmlMetadataTemplate(doiMetadata);
-        
+
         Dataset d = new Dataset();
         GlobalId doi = new GlobalId("doi", "10.5072", "FK2/ABCDEF", null, null, null);
         d.setGlobalId(doi);
@@ -159,11 +161,12 @@ public void testDataCiteXMLCreation() throws IOException {
         try {
             StreamSource source = new StreamSource(new StringReader(xml));
             source.setSystemId("DataCite XML for test dataset");
-            assertTrue(XmlValidator.validateXmlSchema(source, new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd")));
+            assertTrue(XmlValidator.validateXmlSchema(source,
+                    new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd")));
         } catch (SAXException e) {
             System.out.println("Invalid schema: " + e.getMessage());
         }
-        
+
     }
 
     /**
@@ -189,38 +192,39 @@ public void testDataCiteXMLCreationAllFields() throws IOException {
         dv.setVersionState(VersionState.DRAFT);
 
         testDatasetField.setDatasetVersion(dv);
-        
+
         File datasetVersionJson = new File("src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt");
         String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath())));
         JsonObject datasetJson = JsonUtil.getJsonObject(datasetVersionAsJson);
-        
-        GlobalId doi = new GlobalId("doi", datasetJson.getString("authority"), datasetJson.getString("identifier"), null, null, null);
+
+        GlobalId doi = new GlobalId("doi", datasetJson.getString("authority"), datasetJson.getString("identifier"),
+                null, null, null);
         d.setGlobalId(doi);
 
-        
-        List<DatasetField> fields = assertDoesNotThrow(() -> XmlMetadataTemplateTest.parseMetadataBlocks(datasetJson.getJsonObject("datasetVersion").getJsonObject("metadataBlocks")));
+        List<DatasetField> fields = assertDoesNotThrow(() -> XmlMetadataTemplateTest
+                .parseMetadataBlocks(datasetJson.getJsonObject("datasetVersion").getJsonObject("metadataBlocks")));
         dv.setDatasetFields(fields);
-        
+
         ArrayList<DatasetVersion> dsvs = new ArrayList<>();
         dsvs.add(0, dv);
         d.setVersions(dsvs);
         DatasetType dType = new DatasetType();
         dType.setName(DatasetType.DATASET_TYPE_DATASET);
         d.setDatasetType(dType);
-        String xml = DOIDataCiteRegisterService.getMetadataFromDvObject(
-                dv.getDataset().getGlobalId().asString(), new DataCitation(dv).getDataCiteMetadata(), dv.getDataset());
+        String xml = DOIDataCiteRegisterService.getMetadataFromDvObject(dv.getDataset().getGlobalId().asString(),
+                new DataCitation(dv).getDataCiteMetadata(), dv.getDataset());
         System.out.println("Output from dataset-all-defaults is " + xml);
         try {
             StreamSource source = new StreamSource(new StringReader(xml));
             source.setSystemId("DataCite XML for test dataset");
-            assertTrue(XmlValidator.validateXmlSchema(source, new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd")));
+            assertTrue(XmlValidator.validateXmlSchema(source,
+                    new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd")));
         } catch (SAXException e) {
             System.out.println("Invalid schema: " + e.getMessage());
         }
-        
+
     }
 
-    
     /**
      * Mock Utility Methods - These methods support importing DatasetFields from the
      * Dataverse JSON export format. They assume that any DatasetFieldType
@@ -230,9 +234,9 @@ public void testDataCiteXMLCreationAllFields() throws IOException {
      * references and DatasetFieldType-related error checking removed.
      */
     public static List<DatasetField> parseMetadataBlocks(JsonObject json) throws JsonParseException {
-        
+
         Map<String, DatasetFieldType> existingTypes = new HashMap<>();
-        
+
         Set<String> keys = json.keySet();
         List<DatasetField> fields = new LinkedList<>();
 
@@ -259,10 +263,10 @@ private static List<DatasetField> parseFieldsFromArray(JsonArray fieldsArray, Bo
         }
         return fields;
 
-    }    
+    }
 
-    
-    public  static DatasetField parseField(JsonObject json, Boolean testType,  MetadataBlock block, Map<String, DatasetFieldType> existingTypes) throws JsonParseException {
+    public static DatasetField parseField(JsonObject json, Boolean testType, MetadataBlock block,
+            Map<String, DatasetFieldType> existingTypes) throws JsonParseException {
         if (json == null) {
             return null;
         }
@@ -270,7 +274,7 @@ public  static DatasetField parseField(JsonObject json, Boolean testType,  Metad
         DatasetField ret = new DatasetField();
         String fieldName = json.getString("typeName", "");
         String typeClass = json.getString("typeClass", "");
-        if(!existingTypes.containsKey(fieldName)) {
+        if (!existingTypes.containsKey(fieldName)) {
             boolean multiple = json.getBoolean("multiple");
             DatasetFieldType fieldType = new DatasetFieldType();
             fieldType.setName(fieldName);
@@ -294,8 +298,10 @@ public  static DatasetField parseField(JsonObject json, Boolean testType,  Metad
 
         return ret;
     }
-    
-    public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json, Boolean testType, MetadataBlock block, Map<String, DatasetFieldType> existingTypes) throws JsonParseException {
+
+    public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json,
+            Boolean testType, MetadataBlock block, Map<String, DatasetFieldType> existingTypes)
+            throws JsonParseException {
         List<ControlledVocabularyException> vocabExceptions = new ArrayList<>();
         List<DatasetFieldCompoundValue> vals = new LinkedList<>();
         if (compoundType.isAllowMultiples()) {
@@ -303,23 +309,24 @@ public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoun
             try {
                 json.getJsonArray("value").getValuesAs(JsonObject.class);
             } catch (ClassCastException cce) {
-                throw new JsonParseException("Invalid values submitted for " + compoundType.getName() + ". It should be an array of values.");
+                throw new JsonParseException("Invalid values submitted for " + compoundType.getName()
+                        + ". It should be an array of values.");
             }
             for (JsonObject obj : json.getJsonArray("value").getValuesAs(JsonObject.class)) {
                 DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue();
                 List<DatasetField> fields = new LinkedList<>();
                 for (String fieldName : obj.keySet()) {
                     JsonObject childFieldJson = obj.getJsonObject(fieldName);
-                    DatasetField f=null;
+                    DatasetField f = null;
                     try {
                         f = parseField(childFieldJson, testType, block, existingTypes);
-                    } catch(ControlledVocabularyException ex) {
+                    } catch (ControlledVocabularyException ex) {
                         vocabExceptions.add(ex);
                     }
-                    
-                    if (f!=null) {
+
+                    if (f != null) {
                         f.setParentDatasetFieldCompoundValue(cv);
-                            fields.add(f);
+                        fields.add(f);
                     }
                 }
                 if (!fields.isEmpty()) {
@@ -330,10 +337,8 @@ public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoun
                 order++;
             }
 
-           
-
         } else {
-            
+
             DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue();
             List<DatasetField> fields = new LinkedList<>();
             JsonObject value = json.getJsonObject("value");
@@ -341,11 +346,11 @@ public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoun
                 JsonObject childFieldJson = value.getJsonObject(key);
                 DatasetField f = null;
                 try {
-                    f=parseField(childFieldJson, testType, block, existingTypes);
-                } catch(ControlledVocabularyException ex ) {
+                    f = parseField(childFieldJson, testType, block, existingTypes);
+                } catch (ControlledVocabularyException ex) {
                     vocabExceptions.add(ex);
                 }
-                if (f!=null) {
+                if (f != null) {
                     f.setParentDatasetFieldCompoundValue(cv);
                     fields.add(f);
                 }
@@ -354,10 +359,11 @@ public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoun
                 cv.setChildDatasetFields(fields);
                 vals.add(cv);
             }
-      
-    }
+
+        }
         if (!vocabExceptions.isEmpty()) {
-            throw new CompoundVocabularyException( "Invalid controlled vocabulary in compound field ", vocabExceptions, vals);
+            throw new CompoundVocabularyException("Invalid controlled vocabulary in compound field ", vocabExceptions,
+                    vals);
         }
 
         for (DatasetFieldCompoundValue dsfcv : vals) {
@@ -366,13 +372,15 @@ public static void parseCompoundValue(DatasetField dsf, DatasetFieldType compoun
         dsf.setDatasetFieldCompoundValues(vals);
     }
 
-    public static void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObject json) throws JsonParseException {
+    public static void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft, JsonObject json)
+            throws JsonParseException {
         List<DatasetFieldValue> vals = new LinkedList<>();
         if (dft.isAllowMultiples()) {
-           try {
-            json.getJsonArray("value").getValuesAs(JsonObject.class);
+            try {
+                json.getJsonArray("value").getValuesAs(JsonObject.class);
             } catch (ClassCastException cce) {
-                throw new JsonParseException("Invalid values submitted for " + dft.getName() + ". It should be an array of values.");
+                throw new JsonParseException(
+                        "Invalid values submitted for " + dft.getName() + ". It should be an array of values.");
             }
             for (JsonString val : json.getJsonArray("value").getValuesAs(JsonString.class)) {
                 DatasetFieldValue datasetFieldValue = new DatasetFieldValue(dsf);
@@ -382,10 +390,12 @@ public static void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft ,
             }
 
         } else {
-            try {json.getString("value");}
-            catch (ClassCastException cce) {
-                throw new JsonParseException("Invalid value submitted for " + dft.getName() + ". It should be a single value.");
-            }            
+            try {
+                json.getString("value");
+            } catch (ClassCastException cce) {
+                throw new JsonParseException(
+                        "Invalid value submitted for " + dft.getName() + ". It should be a single value.");
+            }
             DatasetFieldValue datasetFieldValue = new DatasetFieldValue();
             datasetFieldValue.setValue(json.getString("value", "").trim());
             datasetFieldValue.setDatasetField(dsf);
@@ -394,15 +404,17 @@ public static void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft ,
 
         dsf.setDatasetFieldValues(vals);
     }
-    
-    public static void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cvvType, JsonObject json) throws JsonParseException {
+
+    public static void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cvvType, JsonObject json)
+            throws JsonParseException {
         List<ControlledVocabularyValue> vals = new LinkedList<>();
         try {
             if (cvvType.isAllowMultiples()) {
                 try {
                     json.getJsonArray("value").getValuesAs(JsonObject.class);
                 } catch (ClassCastException cce) {
-                    throw new JsonParseException("Invalid values submitted for " + cvvType.getName() + ". It should be an array of values.");
+                    throw new JsonParseException(
+                            "Invalid values submitted for " + cvvType.getName() + ". It should be an array of values.");
                 }
                 for (JsonString strVal : json.getJsonArray("value").getValuesAs(JsonString.class)) {
                     String strValue = strVal.getString();
@@ -416,7 +428,8 @@ public static void parseControlledVocabularyValue(DatasetField dsf, DatasetField
                 try {
                     json.getString("value");
                 } catch (ClassCastException cce) {
-                    throw new JsonParseException("Invalid value submitted for " + cvvType.getName() + ". It should be a single value.");
+                    throw new JsonParseException(
+                            "Invalid value submitted for " + cvvType.getName() + ". It should be a single value.");
                 }
                 String strValue = json.getString("value", "");
                 ControlledVocabularyValue cvv = new ControlledVocabularyValue();

From 8ef8cfd2c70d34f458d5bad33d7b790c3150b409 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 13:35:44 +0100
Subject: [PATCH 076/137] Added: parseDataverseUpdates unit test

---
 .../dataverse/util/json/JsonParserTest.java   | 54 ++++++++++++-------
 1 file changed, 35 insertions(+), 19 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
index 59e175f30c1..1a1d836f6a0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
@@ -4,17 +4,9 @@
 
 package edu.harvard.iq.dataverse.util.json;
 
-import edu.harvard.iq.dataverse.ControlledVocabularyValue;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetField;
-import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
-import edu.harvard.iq.dataverse.DatasetFieldType;
+import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
-import edu.harvard.iq.dataverse.DatasetFieldValue;
-import edu.harvard.iq.dataverse.DatasetVersion;
-import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DataverseTheme.Alignment;
-import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.UserNotification.Type;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroupProvider;
@@ -50,16 +42,7 @@
 import java.io.StringReader;
 import java.math.BigDecimal;
 import java.text.ParseException;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.TimeZone;
+import java.util.*;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.*;
@@ -281,6 +264,39 @@ public void testParseCompleteDataverse() throws JsonParseException {
             throw new JsonParseException("Couldn't read test file", ioe);
         }
     }
+
+    /**
+     * TODO
+     * @throws JsonParseException when this test is broken.
+     */
+    @Test
+    public void parseDataverseUpdates() throws JsonParseException {
+        Dataverse dataverse = new Dataverse();
+            dataverse.setName("Name to update");
+            dataverse.setAlias("aliasToUpdate");
+            dataverse.setAffiliation("Affiliation to update");
+            dataverse.setDescription("Description to update");
+            dataverse.setDataverseType(Dataverse.DataverseType.DEPARTMENT);
+            List<DataverseContact> originalContacts = new ArrayList<>();
+            originalContacts.add(new DataverseContact(dataverse, "updatethis@example.edu"));
+            dataverse.setDataverseContacts(originalContacts);
+        JsonObject dvJson;
+        try (FileReader reader = new FileReader("doc/sphinx-guides/source/_static/api/dataverse-complete.json")) {
+            dvJson = Json.createReader(reader).readObject();
+            Dataverse actual = sut.parseDataverseUpdates(dvJson, dataverse);
+            assertEquals("Scientific Research", actual.getName());
+            assertEquals("science", actual.getAlias());
+            assertEquals("Scientific Research University", actual.getAffiliation());
+            assertEquals("We do all the science.", actual.getDescription());
+            assertEquals("LABORATORY", actual.getDataverseType().toString());
+            assertEquals(2, actual.getDataverseContacts().size());
+            assertEquals("pi@example.edu,student@example.edu", actual.getContactEmails());
+            assertEquals(0, actual.getDataverseContacts().get(0).getDisplayOrder());
+            assertEquals(1, actual.getDataverseContacts().get(1).getDisplayOrder());
+        } catch (IOException ioe) {
+            throw new JsonParseException("Couldn't read test file", ioe);
+        }
+    }
     
     @Test
     public void testParseThemeDataverse() throws JsonParseException {

From 62df2a7d534b87cd8975fd01317d3d1a05576e4a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 14:36:27 +0100
Subject: [PATCH 077/137] Changed: reordered logic in UpdateDataverseCommand
 for further refactoring

---
 .../command/impl/UpdateDataverseCommand.java  | 56 +++++++++++--------
 1 file changed, 34 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index 551f0ffdff7..16b93debb6d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -78,6 +78,11 @@ public UpdateDataverseCommand(Dataverse editedDv,
     public Dataverse execute(CommandContext ctxt) throws CommandException {
         logger.fine("Entering update dataverse command");
 
+        if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
+            editedDv.setMetadataBlockRoot(true);
+            editedDv.setMetadataBlocks(metadataBlocks);
+        }
+
         // Perform any optional validation steps, if defined:
         if (ctxt.systemConfig().isExternalDataverseValidationEnabled()) {
             // For admins, an override of the external validation step may be enabled:
@@ -98,39 +103,46 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
         String oldDvAlias = oldDv.getAlias();
         String oldDvName = oldDv.getName();
 
-        Dataverse result = ctxt.dataverses().save(editedDv);
-
-        if (facetList != null) {
-            ctxt.facets().deleteFacetsFor(result);
-            int i = 0;
-            for (DatasetFieldType df : facetList) {
-                ctxt.facets().create(i++, df.getId(), result.getId());
-            }
+        // We don't want to reindex the children datasets unnecessarily:
+        // When these values are changed we need to reindex all children datasets
+        // This check is not recursive as all the values just report the immediate parent
+        if (!oldDvType.equals(editedDv.getDataverseType())
+                || !oldDvName.equals(editedDv.getName())
+                || !oldDvAlias.equals(editedDv.getAlias())) {
+            datasetsReindexRequired = true;
         }
+
         if (featuredDataverseList != null) {
-            ctxt.featuredDataverses().deleteFeaturedDataversesFor(result);
+            ctxt.featuredDataverses().deleteFeaturedDataversesFor(editedDv);
             int i = 0;
             for (Object obj : featuredDataverseList) {
                 Dataverse dv = (Dataverse) obj;
-                ctxt.featuredDataverses().create(i++, dv.getId(), result.getId());
+                ctxt.featuredDataverses().create(i++, dv.getId(), editedDv.getId());
             }
         }
-        if (inputLevelList != null) {
-            ctxt.fieldTypeInputLevels().deleteFacetsFor(result);
-            for (DataverseFieldTypeInputLevel obj : inputLevelList) {
-                ctxt.fieldTypeInputLevels().create(obj);
+
+        if (facetList != null) {
+            ctxt.facets().deleteFacetsFor(editedDv);
+            if (!facetList.isEmpty()) {
+                editedDv.setFacetRoot(true);
+            }
+            int i = 0;
+            for (DatasetFieldType df : facetList) {
+                ctxt.facets().create(i++, df, editedDv);
             }
         }
-
-        // We don't want to reindex the children datasets unnecessarily:
-        // When these values are changed we need to reindex all children datasets
-        // This check is not recursive as all the values just report the immediate parent
-        if (!oldDvType.equals(editedDv.getDataverseType())
-                || !oldDvName.equals(editedDv.getName())
-                || !oldDvAlias.equals(editedDv.getAlias())) {
-            datasetsReindexRequired = true;
+        if (inputLevelList != null) {
+            if (!inputLevelList.isEmpty()) {
+                editedDv.addInputLevelsMetadataBlocksIfNotPresent(inputLevelList);
+            }
+            ctxt.fieldTypeInputLevels().deleteFacetsFor(editedDv);
+            for (DataverseFieldTypeInputLevel inputLevel : inputLevelList) {
+                inputLevel.setDataverse(editedDv);
+                ctxt.fieldTypeInputLevels().create(inputLevel);
+            }
         }
 
+        Dataverse result = ctxt.dataverses().save(editedDv);
         return result;
     }
 

From 6ccbb4ae53ee6bdd573b686e98c964ecf4e8d2db Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 14:37:05 +0100
Subject: [PATCH 078/137] Changed: updateDataverse return code

---
 src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 0bc389041c2..d8bd2b8cb4b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -188,7 +188,7 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
 
             AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
             updatedDataverse = execCommand(new UpdateDataverseCommand(updatedDataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks));
-            return created("/dataverses/" + updatedDataverse.getAlias(), json(updatedDataverse));
+            return ok(json(updatedDataverse));
 
         } catch (WrappedResponse ww) {
             return handleWrappedResponse(ww);

From 5c1703906dfeb205604dd8608b286ee706295e2d Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 14:37:47 +0100
Subject: [PATCH 079/137] Added: IT for updateDataverse endpoint

---
 .../iq/dataverse/api/DataversesIT.java        | 43 +++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 62 +++++++++++++++++++
 2 files changed, 105 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 8c6a8244af1..7abc35d536a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -1253,6 +1253,49 @@ public void testAddDataverse() {
                 .body("message", equalTo("Invalid metadata block name: \"" + invalidMetadataBlockName + "\""));
     }
 
+    @Test
+    public void testUpdateDataverse() {
+        Response createUser = UtilIT.createRandomUser();
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        String testAliasSuffix = "-update-dataverse";
+
+        String testDataverseAlias = UtilIT.getRandomDvAlias() + testAliasSuffix;
+        Response createSubDataverseResponse = UtilIT.createSubDataverse(testDataverseAlias, null, apiToken, "root");
+        createSubDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+
+        String newAlias = UtilIT.getRandomDvAlias() + testAliasSuffix;
+        String newName = "New Test Dataverse Name";
+        String newAffiliation = "New Test Dataverse Affiliation";
+        String newDataverseType = Dataverse.DataverseType.TEACHING_COURSES.toString();
+        String[] newContactEmails = new String[] {"new_email@dataverse.com"};
+        String[] newInputLevelNames = new String[] {"geographicCoverage"};
+        String[] newFacetIds = new String[] {"contributorName"};
+        String[] newMetadataBlockNames = new String[] {"citation", "geospatial", "biomedical"};
+
+        Response updateDataverseResponse = UtilIT.updateDataverse(
+                testDataverseAlias,
+                newAlias,
+                newName,
+                newAffiliation,
+                newDataverseType,
+                newContactEmails,
+                newInputLevelNames,
+                newFacetIds,
+                newMetadataBlockNames,
+                apiToken
+        );
+
+        updateDataverseResponse.prettyPrint();
+        updateDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // TODO add more assertions and cases
+
+        // The alias has been changed, so we should not be able to do any operation using the old one
+        String oldDataverseAlias = testDataverseAlias;
+        Response getDataverseResponse = UtilIT.listDataverseFacets(oldDataverseAlias, apiToken);
+        getDataverseResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+    }
+
     @Test
     public void testListFacets() {
         Response createUserResponse = UtilIT.createRandomUser();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 70f49d81b35..eb40a85f10c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.api;
 
+import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import io.restassured.http.ContentType;
 import io.restassured.path.json.JsonPath;
 import io.restassured.response.Response;
@@ -12,6 +13,7 @@
 import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObject;
 
+import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
 import static jakarta.ws.rs.core.Response.Status.CREATED;
 
 import java.nio.charset.StandardCharsets;
@@ -428,6 +430,66 @@ static Response createSubDataverse(String alias, String category, String apiToke
         return createDataverseResponse;
     }
 
+    static Response updateDataverse(String alias,
+                                    String newAlias,
+                                    String newName,
+                                    String newAffiliation,
+                                    String newDataverseType,
+                                    String[] newContactEmails,
+                                    String[] newInputLevelNames,
+                                    String[] newFacetIds,
+                                    String[] newMetadataBlockNames,
+                                    String apiToken) {
+        JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder();
+        for(String contactEmail : newContactEmails) {
+            contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", contactEmail));
+        }
+        NullSafeJsonBuilder jsonBuilder = jsonObjectBuilder()
+                .add("alias", newAlias)
+                .add("name", newName)
+                .add("affiliation", newAffiliation)
+                .add("dataverseContacts", contactArrayBuilder)
+                .add("dataverseType", newDataverseType)
+                .add("affiliation", newAffiliation);
+
+        JsonObjectBuilder metadataBlocksObjectBuilder = Json.createObjectBuilder();
+
+        if (newInputLevelNames != null) {
+            JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder();
+            for(String inputLevelName : newInputLevelNames) {
+                inputLevelsArrayBuilder.add(Json.createObjectBuilder()
+                        .add("datasetFieldTypeName", inputLevelName)
+                        .add("required", true)
+                        .add("include", true)
+                );
+            }
+            metadataBlocksObjectBuilder.add("inputLevels", inputLevelsArrayBuilder);
+        }
+
+        if (newMetadataBlockNames != null) {
+            JsonArrayBuilder metadataBlockNamesArrayBuilder = Json.createArrayBuilder();
+            for(String metadataBlockName : newMetadataBlockNames) {
+                metadataBlockNamesArrayBuilder.add(metadataBlockName);
+            }
+            metadataBlocksObjectBuilder.add("metadataBlockNames", metadataBlockNamesArrayBuilder);
+        }
+
+        if (newFacetIds != null) {
+            JsonArrayBuilder facetIdsArrayBuilder = Json.createArrayBuilder();
+            for(String facetId : newFacetIds) {
+                facetIdsArrayBuilder.add(facetId);
+            }
+            metadataBlocksObjectBuilder.add("facetIds", facetIdsArrayBuilder);
+        }
+
+        jsonBuilder.add("metadataBlocks", metadataBlocksObjectBuilder);
+
+        JsonObject dvData = jsonBuilder.build();
+        return given()
+                .body(dvData.toString()).contentType(ContentType.JSON)
+                .when().put("/api/dataverses/" + alias + "?key=" + apiToken);
+    }
+
     static Response createDataverse(JsonObject dvData, String apiToken) {
         Response createDataverseResponse = given()
                 .body(dvData.toString()).contentType(ContentType.JSON)

From e5cdb106e22064fe4fc84fa834dae2bf984525ff Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 14:52:12 +0100
Subject: [PATCH 080/137] Refactor: UtilIT duplication on dataverse write
 operations

---
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 66 ++++++-------------
 1 file changed, 21 insertions(+), 45 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index eb40a85f10c..502f1ecb0a8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -391,43 +391,12 @@ static Response createSubDataverse(String alias, String category, String apiToke
             objectBuilder.add("affiliation", affiliation);
         }
 
-        JsonObjectBuilder metadataBlocksObjectBuilder = Json.createObjectBuilder();
-
-        if (inputLevelNames != null) {
-            JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder();
-            for(String inputLevelName : inputLevelNames) {
-                inputLevelsArrayBuilder.add(Json.createObjectBuilder()
-                        .add("datasetFieldTypeName", inputLevelName)
-                        .add("required", true)
-                        .add("include", true)
-                );
-            }
-            metadataBlocksObjectBuilder.add("inputLevels", inputLevelsArrayBuilder);
-        }
-
-        if (metadataBlockNames != null) {
-            JsonArrayBuilder metadataBlockNamesArrayBuilder = Json.createArrayBuilder();
-            for(String metadataBlockName : metadataBlockNames) {
-                metadataBlockNamesArrayBuilder.add(metadataBlockName);
-            }
-            metadataBlocksObjectBuilder.add("metadataBlockNames", metadataBlockNamesArrayBuilder);
-        }
-
-        if (facetIds != null) {
-            JsonArrayBuilder facetIdsArrayBuilder = Json.createArrayBuilder();
-            for(String facetId : facetIds) {
-                facetIdsArrayBuilder.add(facetId);
-            }
-            metadataBlocksObjectBuilder.add("facetIds", facetIdsArrayBuilder);
-        }
-
-        objectBuilder.add("metadataBlocks", metadataBlocksObjectBuilder);
+        updateDataverseRequestJsonWithMetadataBlocksConfiguration(inputLevelNames, facetIds, metadataBlockNames, objectBuilder);
 
         JsonObject dvData = objectBuilder.build();
-        Response createDataverseResponse = given()
+        return given()
                 .body(dvData.toString()).contentType(ContentType.JSON)
                 .when().post("/api/dataverses/" + parentDV + "?key=" + apiToken);
-        return createDataverseResponse;
     }
 
     static Response updateDataverse(String alias,
@@ -452,11 +421,23 @@ static Response updateDataverse(String alias,
                 .add("dataverseType", newDataverseType)
                 .add("affiliation", newAffiliation);
 
+        updateDataverseRequestJsonWithMetadataBlocksConfiguration(newInputLevelNames, newFacetIds, newMetadataBlockNames, jsonBuilder);
+
+        JsonObject dvData = jsonBuilder.build();
+        return given()
+                .body(dvData.toString()).contentType(ContentType.JSON)
+                .when().put("/api/dataverses/" + alias + "?key=" + apiToken);
+    }
+
+    private static void updateDataverseRequestJsonWithMetadataBlocksConfiguration(String[] inputLevelNames,
+                                                                                  String[] facetIds,
+                                                                                  String[] metadataBlockNames,
+                                                                                  JsonObjectBuilder objectBuilder) {
         JsonObjectBuilder metadataBlocksObjectBuilder = Json.createObjectBuilder();
 
-        if (newInputLevelNames != null) {
+        if (inputLevelNames != null) {
             JsonArrayBuilder inputLevelsArrayBuilder = Json.createArrayBuilder();
-            for(String inputLevelName : newInputLevelNames) {
+            for(String inputLevelName : inputLevelNames) {
                 inputLevelsArrayBuilder.add(Json.createObjectBuilder()
                         .add("datasetFieldTypeName", inputLevelName)
                         .add("required", true)
@@ -466,28 +447,23 @@ static Response updateDataverse(String alias,
             metadataBlocksObjectBuilder.add("inputLevels", inputLevelsArrayBuilder);
         }
 
-        if (newMetadataBlockNames != null) {
+        if (metadataBlockNames != null) {
             JsonArrayBuilder metadataBlockNamesArrayBuilder = Json.createArrayBuilder();
-            for(String metadataBlockName : newMetadataBlockNames) {
+            for(String metadataBlockName : metadataBlockNames) {
                 metadataBlockNamesArrayBuilder.add(metadataBlockName);
             }
             metadataBlocksObjectBuilder.add("metadataBlockNames", metadataBlockNamesArrayBuilder);
         }
 
-        if (newFacetIds != null) {
+        if (facetIds != null) {
             JsonArrayBuilder facetIdsArrayBuilder = Json.createArrayBuilder();
-            for(String facetId : newFacetIds) {
+            for(String facetId : facetIds) {
                 facetIdsArrayBuilder.add(facetId);
             }
             metadataBlocksObjectBuilder.add("facetIds", facetIdsArrayBuilder);
         }
 
-        jsonBuilder.add("metadataBlocks", metadataBlocksObjectBuilder);
-
-        JsonObject dvData = jsonBuilder.build();
-        return given()
-                .body(dvData.toString()).contentType(ContentType.JSON)
-                .when().put("/api/dataverses/" + alias + "?key=" + apiToken);
+        objectBuilder.add("metadataBlocks", metadataBlocksObjectBuilder);
     }
 
     static Response createDataverse(JsonObject dvData, String apiToken) {

From 8020d50c26a3d41bef41403984495bad535dbad2 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 14:56:24 +0100
Subject: [PATCH 081/137] Added: pending doc comment to JsonParserTest method

---
 .../edu/harvard/iq/dataverse/util/json/JsonParserTest.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
index 1a1d836f6a0..2cffa7d921c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
@@ -266,7 +266,8 @@ public void testParseCompleteDataverse() throws JsonParseException {
     }
 
     /**
-     * TODO
+     * Test that a JSON object passed for a complete Dataverse update is correctly parsed.
+     * This checks that all properties are parsed into the correct dataverse properties.
      * @throws JsonParseException when this test is broken.
      */
     @Test

From 2d10f22de0a20ab73ef146d6b90f1fb587672f2a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 17 Oct 2024 15:49:36 +0100
Subject: [PATCH 082/137] Added: missing IT for updateDataverse endpoint

---
 .../iq/dataverse/api/DataversesIT.java        | 63 ++++++++++++++++---
 1 file changed, 55 insertions(+), 8 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 7abc35d536a..c311fa1016e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -1,12 +1,15 @@
 package edu.harvard.iq.dataverse.api;
 
 import io.restassured.RestAssured;
+
 import static io.restassured.RestAssured.given;
 import static io.restassured.path.json.JsonPath.with;
+
 import io.restassured.response.Response;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
@@ -14,6 +17,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.logging.Logger;
+
 import jakarta.json.Json;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
@@ -31,6 +35,7 @@
 import static org.junit.jupiter.api.Assertions.*;
 
 import java.nio.file.Files;
+
 import io.restassured.path.json.JsonPath;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matchers;
@@ -43,7 +48,7 @@ public class DataversesIT {
     public static void setUpClass() {
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
     }
-    
+
     @AfterAll
     public static void afterClass() {
         Response removeExcludeEmail = UtilIT.deleteSetting(SettingsServiceBean.Key.ExcludeEmailFromExport);
@@ -1267,10 +1272,10 @@ public void testUpdateDataverse() {
         String newName = "New Test Dataverse Name";
         String newAffiliation = "New Test Dataverse Affiliation";
         String newDataverseType = Dataverse.DataverseType.TEACHING_COURSES.toString();
-        String[] newContactEmails = new String[] {"new_email@dataverse.com"};
-        String[] newInputLevelNames = new String[] {"geographicCoverage"};
-        String[] newFacetIds = new String[] {"contributorName"};
-        String[] newMetadataBlockNames = new String[] {"citation", "geospatial", "biomedical"};
+        String[] newContactEmails = new String[]{"new_email@dataverse.com"};
+        String[] newInputLevelNames = new String[]{"geographicCoverage"};
+        String[] newFacetIds = new String[]{"contributorName"};
+        String[] newMetadataBlockNames = new String[]{"citation", "geospatial", "biomedical"};
 
         Response updateDataverseResponse = UtilIT.updateDataverse(
                 testDataverseAlias,
@@ -1285,15 +1290,57 @@ public void testUpdateDataverse() {
                 apiToken
         );
 
-        updateDataverseResponse.prettyPrint();
+        // Assert dataverse properties are updated
         updateDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // TODO add more assertions and cases
+        String actualDataverseAlias = updateDataverseResponse.then().extract().path("data.alias");
+        assertEquals(newAlias, actualDataverseAlias);
+        String actualDataverseName = updateDataverseResponse.then().extract().path("data.name");
+        assertEquals(newName, actualDataverseName);
+        String actualDataverseAffiliation = updateDataverseResponse.then().extract().path("data.affiliation");
+        assertEquals(newAffiliation, actualDataverseAffiliation);
+        String actualDataverseType = updateDataverseResponse.then().extract().path("data.dataverseType");
+        assertEquals(newDataverseType, actualDataverseType);
+        String actualContactEmail = updateDataverseResponse.then().extract().path("data.dataverseContacts[0].contactEmail");
+        assertEquals("new_email@dataverse.com", actualContactEmail);
+
+        // Assert metadata blocks are updated
+        Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(newAlias, false, false, apiToken);
+        String actualDataverseMetadataBlock1 = listMetadataBlocksResponse.then().extract().path("data[0].name");
+        String actualDataverseMetadataBlock2 = listMetadataBlocksResponse.then().extract().path("data[1].name");
+        String actualDataverseMetadataBlock3 = listMetadataBlocksResponse.then().extract().path("data[2].name");
+        assertThat(newMetadataBlockNames, hasItemInArray(actualDataverseMetadataBlock1));
+        assertThat(newMetadataBlockNames, hasItemInArray(actualDataverseMetadataBlock2));
+        assertThat(newMetadataBlockNames, hasItemInArray(actualDataverseMetadataBlock3));
+
+        // Assert custom facets are updated
+        Response listDataverseFacetsResponse = UtilIT.listDataverseFacets(newAlias, apiToken);
+        String actualFacetName = listDataverseFacetsResponse.then().extract().path("data[0]");
+        assertThat(newFacetIds, hasItemInArray(actualFacetName));
+
+        // Assert input levels are updated
+        Response listDataverseInputLevelsResponse = UtilIT.listDataverseInputLevels(newAlias, apiToken);
+        String actualInputLevelName = listDataverseInputLevelsResponse.then().extract().path("data[0].datasetFieldTypeName");
+        assertThat(newInputLevelNames, hasItemInArray(actualInputLevelName));
 
         // The alias has been changed, so we should not be able to do any operation using the old one
         String oldDataverseAlias = testDataverseAlias;
         Response getDataverseResponse = UtilIT.listDataverseFacets(oldDataverseAlias, apiToken);
         getDataverseResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // Should return error when the dataverse to edit does not exist
+        updateDataverseResponse = UtilIT.updateDataverse(
+                "unexistingDataverseAlias",
+                newAlias,
+                newName,
+                newAffiliation,
+                newDataverseType,
+                newContactEmails,
+                newInputLevelNames,
+                newFacetIds,
+                newMetadataBlockNames,
+                apiToken
+        );
+        updateDataverseResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
     }
 
     @Test

From 08540194926f4e464515405703222e43d725db74 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 18 Oct 2024 15:45:32 -0400
Subject: [PATCH 083/137] how to set up languages in Docker #10939

Also, explain that Payara must be restarted after loading langs.
---
 doc/release-notes/10939-i18n-docker.md          |  5 +++++
 .../source/container/running/demo.rst           | 17 +++++++++++++++++
 .../source/installation/config.rst              |  2 +-
 docker-compose-dev.yml                          |  1 +
 docker/compose/demo/compose.yml                 |  1 +
 5 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 doc/release-notes/10939-i18n-docker.md

diff --git a/doc/release-notes/10939-i18n-docker.md b/doc/release-notes/10939-i18n-docker.md
new file mode 100644
index 00000000000..d9887b684db
--- /dev/null
+++ b/doc/release-notes/10939-i18n-docker.md
@@ -0,0 +1,5 @@
+## Multiple Language in Docker
+
+Configuration and documentation has been added to explain how to set up multiple languages (e.g. English and French) in the tutorial for setting up Dataverse in Docker.
+
+See also #10939
diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst
index f9642347558..2e404e7a09a 100644
--- a/doc/sphinx-guides/source/container/running/demo.rst
+++ b/doc/sphinx-guides/source/container/running/demo.rst
@@ -137,6 +137,23 @@ In the example below of configuring :ref:`:FooterCopyright` we use the default u
 
 One you make this change it should be visible in the copyright in the bottom left of every page.
 
+Multiple Languages
+++++++++++++++++++
+
+Generally speaking, you'll want to follow :ref:`i18n` in the Installation Guide to set up multiple languages such as English and French.
+
+To set up the toggle between English and French, we'll use a slight variation on the command in the instructions above, adding the unblock key we created above:
+
+``curl "http://localhost:8080/api/admin/settings/:Languages?unblock-key=unblockme" -X PUT -d '[{"locale":"en","title":"English"},{"locale":"fr","title":"Français"}]'``
+
+Similarly, when loading the "languages.zip" file, we'll add the unblock key:
+
+``curl "http://localhost:8080/api/admin/datasetfield/loadpropertyfiles?unblock-key=unblockme" -X POST --upload-file /tmp/languages/languages.zip -H "Content-Type: application/zip"``
+
+Stop and start the Dataverse container in order for the language toggle to work.
+
+Note that ``dataverse.lang.directory=/dv/lang`` has already been configured for you in the ``compose.yml`` file. The step where you loaded "languages.zip" should have populated the ``/dv/lang`` directory with files ending in ".properties".
+
 Next Steps
 ----------
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index e98ed8f5189..a2c27598b76 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1783,7 +1783,7 @@ Now that you have a "languages.zip" file, you can load it into your Dataverse in
 
 ``curl http://localhost:8080/api/admin/datasetfield/loadpropertyfiles -X POST --upload-file /tmp/languages/languages.zip -H "Content-Type: application/zip"``
 
-Click on the languages using the drop down in the header to try them out.
+Stop and start Payara and then click on the languages using the drop down in the header to try them out.
 
 .. _help-translate:
 
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 402a95c0e16..384b70b7a7b 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -57,6 +57,7 @@ services:
         -Ddataverse.pid.fake.label=FakeDOIProvider
         -Ddataverse.pid.fake.authority=10.5072
         -Ddataverse.pid.fake.shoulder=FK2/
+        -Ddataverse.lang.directory=/dv/lang
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4949:4848" # HTTPS (Payara Admin Console)
diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml
index 33e7b52004b..d599967919e 100644
--- a/docker/compose/demo/compose.yml
+++ b/docker/compose/demo/compose.yml
@@ -26,6 +26,7 @@ services:
         -Ddataverse.pid.fake.label=FakeDOIProvider
         -Ddataverse.pid.fake.authority=10.5072
         -Ddataverse.pid.fake.shoulder=FK2/
+        -Ddataverse.lang.directory=/dv/lang
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4848:4848" # HTTP (Payara Admin Console)

From d334b689437e06cf674a6725600c705628845c47 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 21 Oct 2024 09:41:55 +0200
Subject: [PATCH 084/137] Refactor: CreateDataverseCommand inheriting
 AbstractWriteDataverseCommand

---
 .../impl/AbstractWriteDataverseCommand.java   |  84 +++++++++++++++
 .../command/impl/CreateDataverseCommand.java  | 102 +++++-------------
 2 files changed, 110 insertions(+), 76 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
new file mode 100644
index 00000000000..577f877db41
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
@@ -0,0 +1,84 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.*;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TODO
+ */
+abstract class AbstractWriteDataverseCommand extends AbstractCommand<Dataverse> {
+
+    protected Dataverse dataverse;
+    private final List<DataverseFieldTypeInputLevel> inputLevels;
+    private final List<DatasetFieldType> facets;
+    protected final List<MetadataBlock> metadataBlocks;
+
+    public AbstractWriteDataverseCommand(Dataverse dataverse,
+                                         DataverseRequest request,
+                                         List<DatasetFieldType> facets,
+                                         List<DataverseFieldTypeInputLevel> inputLevels,
+                                         List<MetadataBlock> metadataBlocks) {
+        super(request, dataverse.getOwner());
+        this.dataverse = dataverse;
+        if (facets != null) {
+            this.facets = new ArrayList<>(facets);
+        } else {
+            this.facets = null;
+        }
+        if (inputLevels != null) {
+            this.inputLevels = new ArrayList<>(inputLevels);
+        } else {
+            this.inputLevels = null;
+        }
+        if (metadataBlocks != null) {
+            this.metadataBlocks = new ArrayList<>(metadataBlocks);
+        } else {
+            this.metadataBlocks = null;
+        }
+    }
+
+    @Override
+    public Dataverse execute(CommandContext ctxt) throws CommandException {
+        dataverse = innerExecute(ctxt);
+
+        if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
+            dataverse.setMetadataBlockRoot(true);
+            dataverse.setMetadataBlocks(metadataBlocks);
+        }
+
+        if (facets != null) {
+            ctxt.facets().deleteFacetsFor(dataverse);
+
+            if (!facets.isEmpty()) {
+                dataverse.setFacetRoot(true);
+            }
+
+            int i = 0;
+            for (DatasetFieldType df : facets) {
+                ctxt.facets().create(i++, df, dataverse);
+            }
+        }
+
+        if (inputLevels != null) {
+            if (!inputLevels.isEmpty()) {
+                dataverse.addInputLevelsMetadataBlocksIfNotPresent(inputLevels);
+            }
+            ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse);
+            for (DataverseFieldTypeInputLevel inputLevel : inputLevels) {
+                inputLevel.setDataverse(dataverse);
+                ctxt.fieldTypeInputLevels().create(inputLevel);
+            }
+        }
+
+        return ctxt.dataverses().save(dataverse);
+    }
+
+    abstract protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException;
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
index 6957dac416d..ce922dc565d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
@@ -6,11 +6,9 @@
 import edu.harvard.iq.dataverse.authorization.groups.Group;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
-import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
-import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
@@ -27,47 +25,26 @@
  * @author michael
  */
 @RequiredPermissions(Permission.AddDataverse)
-public class CreateDataverseCommand extends AbstractCommand<Dataverse> {
-
-    private final Dataverse created;
-    private final List<DataverseFieldTypeInputLevel> inputLevelList;
-    private final List<DatasetFieldType> facetList;
-    private final List<MetadataBlock> metadataBlocks;
+public class CreateDataverseCommand extends AbstractWriteDataverseCommand {
 
     public CreateDataverseCommand(Dataverse created,
-                                  DataverseRequest aRequest,
-                                  List<DatasetFieldType> facetList,
-                                  List<DataverseFieldTypeInputLevel> inputLevelList) {
-        this(created, aRequest, facetList, inputLevelList, null);
+                                  DataverseRequest request,
+                                  List<DatasetFieldType> facets,
+                                  List<DataverseFieldTypeInputLevel> inputLevels) {
+        this(created, request, facets, inputLevels, null);
     }
 
     public CreateDataverseCommand(Dataverse created,
-                                  DataverseRequest aRequest,
-                                  List<DatasetFieldType> facetList,
-                                  List<DataverseFieldTypeInputLevel> inputLevelList,
+                                  DataverseRequest request,
+                                  List<DatasetFieldType> facets,
+                                  List<DataverseFieldTypeInputLevel> inputLevels,
                                   List<MetadataBlock> metadataBlocks) {
-        super(aRequest, created.getOwner());
-        this.created = created;
-        if (facetList != null) {
-            this.facetList = new ArrayList<>(facetList);
-        } else {
-            this.facetList = null;
-        }
-        if (inputLevelList != null) {
-            this.inputLevelList = new ArrayList<>(inputLevelList);
-        } else {
-            this.inputLevelList = null;
-        }
-        if (metadataBlocks != null) {
-            this.metadataBlocks = new ArrayList<>(metadataBlocks);
-        } else {
-            this.metadataBlocks = null;
-        }
+        super(created, request, facets, inputLevels, metadataBlocks);
     }
 
     @Override
-    public Dataverse execute(CommandContext ctxt) throws CommandException {
-        Dataverse owner = created.getOwner();
+    protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException {
+        Dataverse owner = dataverse.getOwner();
         if (owner == null) {
             if (ctxt.dataverses().isRootDataverseExists()) {
                 throw new IllegalCommandException("Root Dataverse already exists. Cannot create another one", this);
@@ -75,44 +52,44 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
         }
 
         if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
-            created.setMetadataBlockRoot(true);
-            created.setMetadataBlocks(metadataBlocks);
+            dataverse.setMetadataBlockRoot(true);
+            dataverse.setMetadataBlocks(metadataBlocks);
         }
 
-        if (created.getCreateDate() == null) {
-            created.setCreateDate(new Timestamp(new Date().getTime()));
+        if (dataverse.getCreateDate() == null) {
+            dataverse.setCreateDate(new Timestamp(new Date().getTime()));
         }
 
-        if (created.getCreator() == null) {
+        if (dataverse.getCreator() == null) {
             final User user = getRequest().getUser();
             if (user.isAuthenticated()) {
-                created.setCreator((AuthenticatedUser) user);
+                dataverse.setCreator((AuthenticatedUser) user);
             } else {
                 throw new IllegalCommandException("Guest users cannot create a Dataverse.", this);
             }
         }
 
-        if (created.getDataverseType() == null) {
-            created.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED);
+        if (dataverse.getDataverseType() == null) {
+            dataverse.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED);
         }
 
-        if (created.getDefaultContributorRole() == null) {
-            created.setDefaultContributorRole(ctxt.roles().findBuiltinRoleByAlias(DataverseRole.EDITOR));
+        if (dataverse.getDefaultContributorRole() == null) {
+            dataverse.setDefaultContributorRole(ctxt.roles().findBuiltinRoleByAlias(DataverseRole.EDITOR));
         }
 
         // @todo for now we are saying all dataverses are permission root
-        created.setPermissionRoot(true);
+        dataverse.setPermissionRoot(true);
 
-        if (ctxt.dataverses().findByAlias(created.getAlias()) != null) {
-            throw new IllegalCommandException("A dataverse with alias " + created.getAlias() + " already exists", this);
+        if (ctxt.dataverses().findByAlias(dataverse.getAlias()) != null) {
+            throw new IllegalCommandException("A dataverse with alias " + dataverse.getAlias() + " already exists", this);
         }
 
-        if (created.getFilePIDsEnabled() != null && !ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
+        if (dataverse.getFilePIDsEnabled() != null && !ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
             throw new IllegalCommandException("File PIDs cannot be enabled per collection", this);
         }
 
         // Save the dataverse
-        Dataverse managedDv = ctxt.dataverses().save(created);
+        Dataverse managedDv = ctxt.dataverses().save(dataverse);
 
         // Find the built in admin role (currently by alias)
         DataverseRole adminRole = ctxt.roles().findBuiltinRoleByAlias(DataverseRole.ADMIN);
@@ -159,33 +136,6 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
         }
 
         managedDv.setPermissionModificationTime(new Timestamp(new Date().getTime()));
-
-        if (facetList != null) {
-            ctxt.facets().deleteFacetsFor(managedDv);
-
-            if (!facetList.isEmpty()) {
-                managedDv.setFacetRoot(true);
-            }
-
-            int i = 0;
-            for (DatasetFieldType df : facetList) {
-                ctxt.facets().create(i++, df, managedDv);
-            }
-        }
-
-        if (inputLevelList != null) {
-            if (!inputLevelList.isEmpty()) {
-                managedDv.addInputLevelsMetadataBlocksIfNotPresent(inputLevelList);
-            }
-            ctxt.fieldTypeInputLevels().deleteFacetsFor(managedDv);
-            for (DataverseFieldTypeInputLevel inputLevel : inputLevelList) {
-                inputLevel.setDataverse(managedDv);
-                ctxt.fieldTypeInputLevels().create(inputLevel);
-            }
-        }
-
-        // TODO: save is called here and above; we likely don't need both
-        managedDv = ctxt.dataverses().save(managedDv);
         return managedDv;
     }
 

From e7782394b037fb6890f785cebd6f12869630c6c6 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 21 Oct 2024 10:57:54 +0200
Subject: [PATCH 085/137] Refactor: UpdateDataverseCommand inheriting
 AbstractWriteDataverseCommand

---
 .../impl/AbstractWriteDataverseCommand.java   |   5 +-
 .../command/impl/CreateDataverseCommand.java  |   2 +-
 .../command/impl/UpdateDataverseCommand.java  | 102 ++++--------------
 3 files changed, 27 insertions(+), 82 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
index 577f877db41..40c2abf5d21 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
@@ -11,7 +11,7 @@
 import java.util.List;
 
 /**
- * TODO
+ * An abstract base class for commands that perform write operations on {@link Dataverse}s.
  */
 abstract class AbstractWriteDataverseCommand extends AbstractCommand<Dataverse> {
 
@@ -21,11 +21,12 @@ abstract class AbstractWriteDataverseCommand extends AbstractCommand<Dataverse>
     protected final List<MetadataBlock> metadataBlocks;
 
     public AbstractWriteDataverseCommand(Dataverse dataverse,
+                                         Dataverse affectedDataverse,
                                          DataverseRequest request,
                                          List<DatasetFieldType> facets,
                                          List<DataverseFieldTypeInputLevel> inputLevels,
                                          List<MetadataBlock> metadataBlocks) {
-        super(request, dataverse.getOwner());
+        super(request, affectedDataverse);
         this.dataverse = dataverse;
         if (facets != null) {
             this.facets = new ArrayList<>(facets);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
index ce922dc565d..145cfb6199c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
@@ -39,7 +39,7 @@ public CreateDataverseCommand(Dataverse created,
                                   List<DatasetFieldType> facets,
                                   List<DataverseFieldTypeInputLevel> inputLevels,
                                   List<MetadataBlock> metadataBlocks) {
-        super(created, request, facets, inputLevels, metadataBlocks);
+        super(created, created.getOwner(), request, facets, inputLevels, metadataBlocks);
     }
 
     @Override
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index 16b93debb6d..14d9e408be8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -6,16 +6,13 @@
 
 import static edu.harvard.iq.dataverse.dataverse.DataverseUtil.validateDataverseMetadataExternally;
 
-import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
-import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.logging.Logger;
 
 /**
  * Update an existing dataverse.
@@ -23,72 +20,41 @@
  * @author michael
  */
 @RequiredPermissions(Permission.EditDataverse)
-public class UpdateDataverseCommand extends AbstractCommand<Dataverse> {
-    private static final Logger logger = Logger.getLogger(UpdateDataverseCommand.class.getName());
-
-    private final Dataverse editedDv;
-    private final List<DatasetFieldType> facetList;
+public class UpdateDataverseCommand extends AbstractWriteDataverseCommand {
     private final List<Dataverse> featuredDataverseList;
-    private final List<DataverseFieldTypeInputLevel> inputLevelList;
-    private final List<MetadataBlock> metadataBlocks;
 
     private boolean datasetsReindexRequired = false;
 
     public UpdateDataverseCommand(Dataverse editedDv,
-                                  List<DatasetFieldType> facetList,
-                                  List<Dataverse> featuredDataverseList,
-                                  DataverseRequest aRequest,
-                                  List<DataverseFieldTypeInputLevel> inputLevelList) {
-        this(editedDv, facetList, featuredDataverseList, aRequest, inputLevelList, null);
+                                  List<DatasetFieldType> facets,
+                                  List<Dataverse> featuredDataverses,
+                                  DataverseRequest request,
+                                  List<DataverseFieldTypeInputLevel> inputLevels) {
+        this(editedDv, facets, featuredDataverses, request, inputLevels, null);
     }
 
     public UpdateDataverseCommand(Dataverse editedDv,
-                                  List<DatasetFieldType> facetList,
-                                  List<Dataverse> featuredDataverseList,
-                                  DataverseRequest aRequest,
-                                  List<DataverseFieldTypeInputLevel> inputLevelList,
+                                  List<DatasetFieldType> facets,
+                                  List<Dataverse> featuredDataverses,
+                                  DataverseRequest request,
+                                  List<DataverseFieldTypeInputLevel> inputLevels,
                                   List<MetadataBlock> metadataBlocks) {
-        super(aRequest, editedDv);
-        this.editedDv = editedDv;
-        // add update template uses this command but does not
-        // update facet list or featured dataverses
-        if (facetList != null) {
-            this.facetList = new ArrayList<>(facetList);
-        } else {
-            this.facetList = null;
-        }
-        if (featuredDataverseList != null) {
-            this.featuredDataverseList = new ArrayList<>(featuredDataverseList);
+        super(editedDv, editedDv, request, facets, inputLevels, metadataBlocks);
+        if (featuredDataverses != null) {
+            this.featuredDataverseList = new ArrayList<>(featuredDataverses);
         } else {
             this.featuredDataverseList = null;
         }
-        if (inputLevelList != null) {
-            this.inputLevelList = new ArrayList<>(inputLevelList);
-        } else {
-            this.inputLevelList = null;
-        }
-        if (metadataBlocks != null) {
-            this.metadataBlocks = new ArrayList<>(metadataBlocks);
-        } else {
-            this.metadataBlocks = null;
-        }
     }
 
     @Override
-    public Dataverse execute(CommandContext ctxt) throws CommandException {
-        logger.fine("Entering update dataverse command");
-
-        if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
-            editedDv.setMetadataBlockRoot(true);
-            editedDv.setMetadataBlocks(metadataBlocks);
-        }
-
+    protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException {
         // Perform any optional validation steps, if defined:
         if (ctxt.systemConfig().isExternalDataverseValidationEnabled()) {
             // For admins, an override of the external validation step may be enabled:
             if (!(getUser().isSuperuser() && ctxt.systemConfig().isExternalValidationAdminOverrideEnabled())) {
                 String executable = ctxt.systemConfig().getDataverseValidationExecutable();
-                boolean result = validateDataverseMetadataExternally(editedDv, executable, getRequest());
+                boolean result = validateDataverseMetadataExternally(dataverse, executable, getRequest());
 
                 if (!result) {
                     String rejectionMessage = ctxt.systemConfig().getDataverseUpdateValidationFailureMsg();
@@ -97,7 +63,7 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
             }
         }
 
-        Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
+        Dataverse oldDv = ctxt.dataverses().find(dataverse.getId());
 
         DataverseType oldDvType = oldDv.getDataverseType();
         String oldDvAlias = oldDv.getAlias();
@@ -106,44 +72,22 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
         // We don't want to reindex the children datasets unnecessarily:
         // When these values are changed we need to reindex all children datasets
         // This check is not recursive as all the values just report the immediate parent
-        if (!oldDvType.equals(editedDv.getDataverseType())
-                || !oldDvName.equals(editedDv.getName())
-                || !oldDvAlias.equals(editedDv.getAlias())) {
+        if (!oldDvType.equals(dataverse.getDataverseType())
+                || !oldDvName.equals(dataverse.getName())
+                || !oldDvAlias.equals(dataverse.getAlias())) {
             datasetsReindexRequired = true;
         }
 
         if (featuredDataverseList != null) {
-            ctxt.featuredDataverses().deleteFeaturedDataversesFor(editedDv);
+            ctxt.featuredDataverses().deleteFeaturedDataversesFor(dataverse);
             int i = 0;
             for (Object obj : featuredDataverseList) {
                 Dataverse dv = (Dataverse) obj;
-                ctxt.featuredDataverses().create(i++, dv.getId(), editedDv.getId());
-            }
-        }
-
-        if (facetList != null) {
-            ctxt.facets().deleteFacetsFor(editedDv);
-            if (!facetList.isEmpty()) {
-                editedDv.setFacetRoot(true);
-            }
-            int i = 0;
-            for (DatasetFieldType df : facetList) {
-                ctxt.facets().create(i++, df, editedDv);
-            }
-        }
-        if (inputLevelList != null) {
-            if (!inputLevelList.isEmpty()) {
-                editedDv.addInputLevelsMetadataBlocksIfNotPresent(inputLevelList);
-            }
-            ctxt.fieldTypeInputLevels().deleteFacetsFor(editedDv);
-            for (DataverseFieldTypeInputLevel inputLevel : inputLevelList) {
-                inputLevel.setDataverse(editedDv);
-                ctxt.fieldTypeInputLevels().create(inputLevel);
+                ctxt.featuredDataverses().create(i++, dv.getId(), dataverse.getId());
             }
         }
 
-        Dataverse result = ctxt.dataverses().save(editedDv);
-        return result;
+        return dataverse;
     }
 
     @Override
@@ -154,7 +98,7 @@ public boolean onSuccess(CommandContext ctxt, Object r) {
         // It appears that we at some point lost some extra logic here, where
         // we only reindex the underlying datasets if one or more of the specific set
         // of fields have been changed (since these values are included in the
-        // indexed solr documents for dataasets). So I'm putting that back. -L.A.
+        // indexed solr documents for datasets). So I'm putting that back. -L.A.
         Dataverse result = (Dataverse) r;
 
         if (datasetsReindexRequired) {

From 2a62c0460c28a3704b26289a65cd00437684a7c6 Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Mon, 21 Oct 2024 15:15:44 +0200
Subject: [PATCH 086/137] performance test

---
 .../command/impl/CreateNewDataFilesTest.java  | 74 +++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
index 1262984eb27..a956b473a4b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
@@ -13,18 +13,26 @@
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
 import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.jetbrains.annotations.NotNull;
+import org.joda.time.DateTime;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.security.SecureRandom;
+import java.text.MessageFormat;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 import static edu.harvard.iq.dataverse.DataFile.ChecksumType.MD5;
 import static org.apache.commons.io.file.FilesUncheck.createDirectories;
@@ -146,6 +154,72 @@ public void execute_rezips_sets_of_shape_files_from_uploaded_zip() throws Except
         }
     }
 
+    @Disabled("Too slow. Intended for manual execution.")
+    @Test
+    @JvmSetting(key = JvmSettings.FILES_DIRECTORY, value = "/tmp/test/CreateNewDataFilesTest/tmp")
+    public void extract_zip_performance() throws Exception {
+        /*
+         Developed to test performance difference between the old implementation with ZipInputStream and the new ZipFile implementation.
+         Play with numbers depending on:
+         - the time you want to spend on this test
+         - how much system stress you want to examine
+        */
+        var nrOfZipFiles = 20;
+        var avgNrOfFilesPerZip = 300;
+        var avgFileLength = 5000;
+
+        var tmpUploadStorage = Path.of("/tmp/test/CreateNewDataFilesTest/tmp/temp");
+        if(tmpUploadStorage.toFile().exists()) {
+            deleteDirectory(tmpUploadStorage);
+        }
+        createDirectories(tmpUploadStorage); // temp in target would choke intellij
+
+        var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+        var random = new SecureRandom();
+        var totalNrOfFiles = 0;
+        var totalFileSize = 0;
+        var tmp = Path.of(Files.createTempDirectory(null).toString());
+        var ctxt = mockCommandContext(mockSysConfig(false, 100000000L, MD5, 10000));
+        try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) {
+            mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");
+            var before = DateTime.now();
+            for (var zipNr = 1; zipNr <= nrOfZipFiles; zipNr++) {
+                // build the zip
+                var zip = tmp.resolve(zipNr + "-data.zip");
+                var nrOfFilesInZip = random.nextInt(avgNrOfFilesPerZip * 2);
+                try (var zipStream = new ZipOutputStream(new FileOutputStream(zip.toFile()))) {
+                    for (var fileInZipNr = 1; fileInZipNr <= nrOfFilesInZip; fileInZipNr++) {
+                        // build content for a file
+                        var stringLength = random.nextInt(avgFileLength * 2 -5);
+                        StringBuilder sb = new StringBuilder(stringLength);
+                        for (int i = 1; i <= stringLength; i++) {// zero length causes buffer underflow
+                            sb.append(chars.charAt(random.nextInt(chars.length())));
+                        }
+                        // add the file to the zip
+                        zipStream.putNextEntry(new ZipEntry(fileInZipNr + ".txt"));
+                        zipStream.write((sb.toString()).getBytes());
+                        zipStream.closeEntry();
+                        totalFileSize += stringLength;
+                    }
+                }
+
+                // upload the zip
+                var result = createCmd(zip.toString(), mockDatasetVersion(), 1000L, 500L)
+                    .execute(ctxt);
+                assertThat(result.getErrors()).hasSize(0);
+                assertThat(result.getDataFiles()).hasSize(nrOfFilesInZip);
+                totalNrOfFiles += nrOfFilesInZip;
+
+                // report after each zip to have some data even when aborting a test that takes too long
+                System.out.println(MessageFormat.format(
+                    "Total time: {0}ms; nr of zips {1} total nr of files {2}; total file size {3}",
+                    DateTime.now().getMillis() - before.getMillis(), zipNr, totalNrOfFiles, totalFileSize
+                ));
+            }
+            assertThat(tmpUploadStorage.toFile().list()).hasSize(totalNrOfFiles);
+        }
+    }
+
     private static @NotNull CreateNewDataFilesCommand createCmd(String name, DatasetVersion dsVersion, long allocatedQuotaLimit, long usedQuotaLimit) throws FileNotFoundException {
         return new CreateNewDataFilesCommand(
             Mockito.mock(DataverseRequest.class),

From 4e90d0c3fe8d501f5810a162c304ce4e3b43a891 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 21 Oct 2024 16:40:43 +0100
Subject: [PATCH 087/137] Added: docs for #10904

---
 doc/sphinx-guides/source/api/native-api.rst | 52 +++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index f8b8620f121..6254742eebb 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -74,6 +74,58 @@ The request JSON supports an optional ``metadataBlocks`` object, with the follow
 
 To obtain an example of how these objects are included in the JSON file, download :download:`dataverse-complete-optional-params.json <../_static/api/dataverse-complete-optional-params.json>` file and modify it to suit your needs.
 
+.. _update-dataverse-api:
+
+Update a Dataverse Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates an existing Dataverse collection using a JSON file following the same structure as the one used in the API for the creation. (see :ref:`create-dataverse-api`).
+
+The steps for updating a Dataverse collection are:
+
+- Prepare a JSON file containing the fields for the properties you want to update. You do not need to include all the properties, only the ones you want to update.
+- Execute a curl command or equivalent.
+
+As an example, you can download :download:`dataverse-complete.json <../_static/api/dataverse-complete.json>` file and modify it to suit your needs. The controlled vocabulary for ``dataverseType`` is the following:
+
+- ``DEPARTMENT``
+- ``JOURNALS``
+- ``LABORATORY``
+- ``ORGANIZATIONS_INSTITUTIONS``
+- ``RESEARCHERS``
+- ``RESEARCH_GROUP``
+- ``RESEARCH_PROJECTS``
+- ``TEACHING_COURSES``
+- ``UNCATEGORIZED``
+
+The curl command below assumes you are using the name "dataverse-complete.json" and that this file is in your current working directory.
+
+Next you need to figure out the alias or database id of the Dataverse collection you want to update.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export DV_ALIAS=dvAlias
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/dataverses/$DV_ALIAS" --upload-file dataverse-complete.json
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/dataverses/dvAlias" --upload-file dataverse-complete.json
+
+You should expect an HTTP 200 response and JSON beginning with "status":"OK" followed by a representation of the updated Dataverse collection.
+
+Same as in :ref:`create-dataverse-api`, the request JSON supports an optional ``metadataBlocks`` object, with the following supported sub-objects:
+
+- ``metadataBlockNames``: The names of the metadata blocks you want to add to the Dataverse collection.
+- ``inputLevels``: The names of the fields in each metadata block for which you want to add a custom configuration regarding their inclusion or requirement when creating and editing datasets in the new Dataverse collection. Note that if the corresponding metadata blocks names are not specified in the ``metadataBlockNames``` field, they will be added automatically to the Dataverse collection.
+- ``facetIds``: The names of the fields to use as facets for browsing datasets and collections in the new Dataverse collection. Note that the order of the facets is defined by their order in the provided JSON array.
+
+To obtain an example of how these objects are included in the JSON file, download :download:`dataverse-complete-optional-params.json <../_static/api/dataverse-complete-optional-params.json>` file and modify it to suit your needs.
+
 .. _view-dataverse:
 
 View a Dataverse Collection

From 6aac751d55375e7433d01d500f38b8be83a7b5bc Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 21 Oct 2024 16:44:09 +0100
Subject: [PATCH 088/137] Added: release notes for #10904

---
 doc/release-notes/10904-edit-dataverse-collection-endpoint.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/release-notes/10904-edit-dataverse-collection-endpoint.md

diff --git a/doc/release-notes/10904-edit-dataverse-collection-endpoint.md b/doc/release-notes/10904-edit-dataverse-collection-endpoint.md
new file mode 100644
index 00000000000..b9256941eea
--- /dev/null
+++ b/doc/release-notes/10904-edit-dataverse-collection-endpoint.md
@@ -0,0 +1 @@
+Adds a new endpoint (`PUT /api/dataverses/<identifier>`) for updating an existing Dataverse collection using a JSON file following the same structure as the one used in the API for the creation.

From 4f98be6a1bcec06ffcada8098e57baf4ea0dd9d2 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 21 Oct 2024 17:37:26 +0100
Subject: [PATCH 089/137] Removed: unnecessary line in updateDataverse endpoint

---
 src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index d8bd2b8cb4b..895d073bb47 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -184,8 +184,6 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
             List<MetadataBlock> metadataBlocks = parseMetadataBlocks(body);
             List<DatasetFieldType> facets = parseFacets(body);
 
-            updatedDataverse.setId(originalDataverse.getId());
-
             AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
             updatedDataverse = execCommand(new UpdateDataverseCommand(updatedDataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks));
             return ok(json(updatedDataverse));

From a8e1e80002010aa83b95add6923029c91a965c86 Mon Sep 17 00:00:00 2001
From: Thrinadh Manubothu <m.thrinadhreddy@gmail.com>
Date: Tue, 22 Oct 2024 07:27:01 +0530
Subject: [PATCH 090/137] Fixed typo tombstone in Bundle.properties

---
 src/main/java/propertyFiles/Bundle.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 5f3e4c33e0b..149e6a7e828 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2065,7 +2065,7 @@ file.deleteFileDialog.multiple.immediate=The file(s) will be deleted after you c
 file.deleteFileDialog.header=Delete Files
 file.deleteFileDialog.failed.tip=Files will not be removed from previously published versions of the dataset.
 file.deaccessionDialog.tip.permanent=Deaccession is <strong>permanent</strong>.
-file.deaccessionDialog.tip=This dataset will no longer be public and a tumbstone will display the reason for deaccessioning. <br>Please read the <a href="guides.dataverse.org/en/latest/user/dataset-management.html?highlight=deaccession#dataset-deaccession">documentation</a> if you have any questions.
+file.deaccessionDialog.tip=This dataset will no longer be public and a tombstone will display the reason for deaccessioning. <br>Please read the <a href="guides.dataverse.org/en/latest/user/dataset-management.html?highlight=deaccession#dataset-deaccession">documentation</a> if you have any questions.
 file.deaccessionDialog.version=Version
 file.deaccessionDialog.reason.question1=Which version(s) do you want to deaccession?
 file.deaccessionDialog.reason.question2=What is the reason for deaccession?

From 255f4196ef053373205f632a460de392a592efb9 Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Tue, 22 Oct 2024 10:02:24 +0200
Subject: [PATCH 091/137] defensive assert; concise time measurement

---
 .../engine/command/impl/CreateNewDataFilesTest.java      | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
index a956b473a4b..aa015dd68a6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
@@ -137,7 +137,7 @@ public void execute_rezips_sets_of_shape_files_from_uploaded_zip() throws Except
             assertThat(result.getErrors()).hasSize(0);
             assertThat(result.getDataFiles().stream().map(dataFile ->
                 (dataFile.getFileMetadata().getDirectoryLabel() + "/" + dataFile.getDisplayName())
-                    .replaceAll(".*temp/shp_[-0-9]*/", "")
+                    .replaceAll(".*temp/[-_shp0-9]*/", "")
             )).containsExactlyInAnyOrder(
                 "dataDir/shape1.zip",
                 "dataDir/shape2/shape2",
@@ -178,11 +178,11 @@ public void extract_zip_performance() throws Exception {
         var random = new SecureRandom();
         var totalNrOfFiles = 0;
         var totalFileSize = 0;
+        var totalTime = 0L;
         var tmp = Path.of(Files.createTempDirectory(null).toString());
         var ctxt = mockCommandContext(mockSysConfig(false, 100000000L, MD5, 10000));
         try (var mockedJHoveFileType = Mockito.mockStatic(JhoveFileType.class)) {
             mockedJHoveFileType.when(JhoveFileType::getJhoveConfigFile).thenReturn("conf/jhove/jhove.conf");
-            var before = DateTime.now();
             for (var zipNr = 1; zipNr <= nrOfZipFiles; zipNr++) {
                 // build the zip
                 var zip = tmp.resolve(zipNr + "-data.zip");
@@ -204,8 +204,11 @@ public void extract_zip_performance() throws Exception {
                 }
 
                 // upload the zip
+                var before = DateTime.now();
                 var result = createCmd(zip.toString(), mockDatasetVersion(), 1000L, 500L)
                     .execute(ctxt);
+                totalTime += DateTime.now().getMillis() - before.getMillis();
+
                 assertThat(result.getErrors()).hasSize(0);
                 assertThat(result.getDataFiles()).hasSize(nrOfFilesInZip);
                 totalNrOfFiles += nrOfFilesInZip;
@@ -213,7 +216,7 @@ public void extract_zip_performance() throws Exception {
                 // report after each zip to have some data even when aborting a test that takes too long
                 System.out.println(MessageFormat.format(
                     "Total time: {0}ms; nr of zips {1} total nr of files {2}; total file size {3}",
-                    DateTime.now().getMillis() - before.getMillis(), zipNr, totalNrOfFiles, totalFileSize
+                    totalTime, zipNr, totalNrOfFiles, totalFileSize
                 ));
             }
             assertThat(tmpUploadStorage.toFile().list()).hasSize(totalNrOfFiles);

From 580ca364305f8d97404a73724416e549553dcf05 Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Tue, 22 Oct 2024 14:10:17 +0200
Subject: [PATCH 092/137] more defensive assert

Intellij shows directory labels like
ewDataFilesTest/tmp/temp/shp_2024-10-22-01-57-21-833/dataDir/extra
possibly different environments have different values
---
 .../command/impl/CreateNewDataFilesTest.java   | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
index aa015dd68a6..f49ebcea39c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesTest.java
@@ -137,16 +137,16 @@ public void execute_rezips_sets_of_shape_files_from_uploaded_zip() throws Except
             assertThat(result.getErrors()).hasSize(0);
             assertThat(result.getDataFiles().stream().map(dataFile ->
                 (dataFile.getFileMetadata().getDirectoryLabel() + "/" + dataFile.getDisplayName())
-                    .replaceAll(".*temp/[-_shp0-9]*/", "")
+                    .replaceAll(".*/dataDir/", "")
             )).containsExactlyInAnyOrder(
-                "dataDir/shape1.zip",
-                "dataDir/shape2/shape2",
-                "dataDir/shape2/shape2.pdf",
-                "dataDir/shape2/shape2.txt",
-                "dataDir/shape2/shape2.zip",
-                "dataDir/extra/shp_dictionary.xls",
-                "dataDir/extra/notes",
-                "dataDir/extra/README.MD"
+                "shape1.zip",
+                "shape2/shape2",
+                "shape2/shape2.pdf",
+                "shape2/shape2.txt",
+                "shape2/shape2.zip",
+                "extra/shp_dictionary.xls",
+                "extra/notes",
+                "extra/README.MD"
             );
             var storageIds = result.getDataFiles().stream().map(DataFile::getStorageIdentifier).toList();
             assertThat(tempDir.toFile().list())

From ba99445f267a0d402c870e9156c1a7a7bc5516b1 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 22 Oct 2024 13:15:29 -0400
Subject: [PATCH 093/137] update xmlunit-assertj3 as well

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index fa99adca8b2..3da3fc070c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -615,7 +615,7 @@
         <dependency>
             <groupId>org.xmlunit</groupId>
             <artifactId>xmlunit-assertj3</artifactId>
-            <version>2.8.2</version>
+            <version>2.10.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>

From c76eddd1a26431fa4233d1522d6a373dae363e40 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Tue, 22 Oct 2024 14:55:02 -0400
Subject: [PATCH 094/137] Update testing-infrastructure.md

After a release, it's important to update the version number in the PR to match the latest release version. Failing to do so can cause the Jenkins build to fail. Ensure that the PR version number aligns with the current Dataverse version to avoid this issue.
---
 doc/sphinx-guides/source/qa/testing-infrastructure.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/qa/testing-infrastructure.md b/doc/sphinx-guides/source/qa/testing-infrastructure.md
index 6ec26c6da49..2055565289d 100644
--- a/doc/sphinx-guides/source/qa/testing-infrastructure.md
+++ b/doc/sphinx-guides/source/qa/testing-infrastructure.md
@@ -31,6 +31,8 @@ To build and test a PR, we use a job called `IQSS_Dataverse_Internal` on <https:
 
 1. If for some reason it didn't deploy, check the server.log file. It may just be a caching issue so try un-deploying, deleting cache, restarting, and re-deploying on the server (`su - dataverse` then `/usr/local/payara6/bin/asadmin list-applications; /usr/local/payara6/bin/asadmin undeploy dataverse-6.1; /usr/local/payara6/bin/asadmin deploy /tmp/dataverse-6.1.war`)
 
+1. When a Jenkins job fails after a release, it might be due to the version number in the `pom.xml` file not being updated in the pull request (PR). To verify this, open the relevant GitHub ticket, navigate to the PR branch, and go to `dataverse > modules > dataverse-parent > pom.xml`. Look for the version number, typically shown as `<revision>6.3</revision>`, and ensure it matches the current Dataverse build version.
+
 1. If that didn't work, you may have run into a Flyway DB script collision error but that should be indicated by the server.log. See {doc}`/developers/sql-upgrade-scripts` in the Developer Guide. In the case of a collision, ask the developer to rename the script.
 
 1.	Assuming the above steps worked, and they should 99% of the time, test away! Note: be sure to `tail -F server.log` in a terminal window while you are doing any testing. This way you can spot problems that may not appear in the UI and have easier access to any stack traces for easier reporting.

From 6c983e829cc6a67002e0aefe17f70a9f7f9b1e99 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Tue, 22 Oct 2024 15:32:31 -0400
Subject: [PATCH 095/137] Update
 doc/sphinx-guides/source/qa/testing-infrastructure.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/qa/testing-infrastructure.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/qa/testing-infrastructure.md b/doc/sphinx-guides/source/qa/testing-infrastructure.md
index 2055565289d..7f7f7c17063 100644
--- a/doc/sphinx-guides/source/qa/testing-infrastructure.md
+++ b/doc/sphinx-guides/source/qa/testing-infrastructure.md
@@ -31,7 +31,7 @@ To build and test a PR, we use a job called `IQSS_Dataverse_Internal` on <https:
 
 1. If for some reason it didn't deploy, check the server.log file. It may just be a caching issue so try un-deploying, deleting cache, restarting, and re-deploying on the server (`su - dataverse` then `/usr/local/payara6/bin/asadmin list-applications; /usr/local/payara6/bin/asadmin undeploy dataverse-6.1; /usr/local/payara6/bin/asadmin deploy /tmp/dataverse-6.1.war`)
 
-1. When a Jenkins job fails after a release, it might be due to the version number in the `pom.xml` file not being updated in the pull request (PR). To verify this, open the relevant GitHub ticket, navigate to the PR branch, and go to `dataverse > modules > dataverse-parent > pom.xml`. Look for the version number, typically shown as `<revision>6.3</revision>`, and ensure it matches the current Dataverse build version.
+1. When a Jenkins job fails after a release, it might be due to the version number in the `pom.xml` file not being updated in the pull request (PR). To verify this, open the relevant GitHub issue, navigate to the PR branch, and go to `dataverse > modules > dataverse-parent > pom.xml`. Look for the version number, typically shown as `<revision>6.3</revision>`, and ensure it matches the current Dataverse build version. If it doesn't match, ask the developer to update the branch with the latest from the "develop" branch.
 
 1. If that didn't work, you may have run into a Flyway DB script collision error but that should be indicated by the server.log. See {doc}`/developers/sql-upgrade-scripts` in the Developer Guide. In the case of a collision, ask the developer to rename the script.
 

From 1b1d337d4c82cfa52ac3777fc99d88de1b2d45b8 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 23 Oct 2024 14:18:28 +0100
Subject: [PATCH 096/137] Changed: using new UpdateDataverseAttributeCommand
 from updateAttribute endpoint

---
 .../harvard/iq/dataverse/api/Dataverses.java  |  62 ++--------
 .../impl/UpdateDataverseAttributeCommand.java | 110 ++++++++++++++++++
 2 files changed, 121 insertions(+), 51 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 0ee146ed99b..110adb60577 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -621,62 +621,22 @@ public Response deleteDataverse(@Context ContainerRequestContext crc, @PathParam
     public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier,
                                     @PathParam("attribute") String attribute, @QueryParam("value") String value) {
         try {
-            Dataverse collection = findDataverseOrDie(identifier);
-            User user = getRequestUser(crc);
-            DataverseRequest dvRequest = createDataverseRequest(user);
-    
-            // TODO: The cases below use hard coded strings, because we have no place for definitions of those!
-            //       They are taken from util.json.JsonParser / util.json.JsonPrinter. This shall be changed.
-            //       This also should be extended to more attributes, like the type, theme, contacts, some booleans, etc.
-            switch (attribute) {
-                case "alias":
-                    collection.setAlias(value);
-                    break;
-                case "name":
-                    collection.setName(value);
-                    break;
-                case "description":
-                    collection.setDescription(value);
-                    break;
-                case "affiliation":
-                    collection.setAffiliation(value);
-                    break;
-                /* commenting out the code from the draft pr #9462:
-                case "versionPidsConduct":
-                    CollectionConduct conduct = CollectionConduct.findBy(value);
-                    if (conduct == null) {
-                        return badRequest("'" + value + "' is not one of [" +
-                            String.join(",", CollectionConduct.asList()) + "]");
-                    }
-                    collection.setDatasetVersionPidConduct(conduct);
-                    break;
-                 */
-                case "filePIDsEnabled":
-                    if(!user.isSuperuser()) {
-                        return forbidden("You must be a superuser to change this setting");
-                    }
-                    if(!settingsService.isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
-                        return forbidden("Changing File PID policy per collection is not enabled on this server");
-                    }
-                    collection.setFilePIDsEnabled(parseBooleanOrDie(value));
-                    break;
-                default:
-                    return badRequest("'" + attribute + "' is not a supported attribute");
-            }
-            
-            // Off to persistence layer
-            execCommand(new UpdateDataverseCommand(collection, null, null, dvRequest, null));
-    
-            // Also return modified collection to user
-            return ok("Update successful", JsonPrinter.json(collection));
-        
-        // TODO: This is an anti-pattern, necessary due to this bean being an EJB, causing very noisy and unnecessary
-        //       logging by the EJB container for bubbling exceptions. (It would be handled by the error handlers.)
+            Dataverse dataverse = findDataverseOrDie(identifier);
+            Object formattedValue = formatAttributeValue(attribute, value);
+            dataverse = execCommand(new UpdateDataverseAttributeCommand(createDataverseRequest(getRequestUser(crc)), dataverse, attribute, formattedValue));
+            return ok(JsonPrinter.json(dataverse));
         } catch (WrappedResponse e) {
             return e.getResponse();
         }
     }
 
+    private Object formatAttributeValue(String attribute, String value) throws WrappedResponse {
+        if (attribute.equals("filePIDsEnabled")) {
+            return parseBooleanOrDie(value);
+        }
+        return value;
+    }
+
     @GET
     @AuthRequired
     @Path("{identifier}/inputLevels")
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
new file mode 100644
index 00000000000..57ac20fcee6
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
@@ -0,0 +1,110 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+
+import java.util.Collections;
+
+/**
+ * Command to update an existing Dataverse attribute.
+ */
+@RequiredPermissions(Permission.EditDataverse)
+public class UpdateDataverseAttributeCommand extends AbstractCommand<Dataverse> {
+
+    private static final String ATTRIBUTE_ALIAS = "alias";
+    private static final String ATTRIBUTE_NAME = "name";
+    private static final String ATTRIBUTE_DESCRIPTION = "description";
+    private static final String ATTRIBUTE_AFFILIATION = "affiliation";
+    private static final String ATTRIBUTE_FILE_PIDS_ENABLED = "filePIDsEnabled";
+
+    private final Dataverse dataverse;
+    private final String attributeName;
+    private final Object attributeValue;
+
+    public UpdateDataverseAttributeCommand(DataverseRequest request, Dataverse dataverse, String attributeName, Object attributeValue) {
+        super(request, dataverse);
+        this.dataverse = dataverse;
+        this.attributeName = attributeName;
+        this.attributeValue = attributeValue;
+    }
+
+    @Override
+    public Dataverse execute(CommandContext ctxt) throws CommandException {
+        switch (attributeName) {
+            case ATTRIBUTE_ALIAS:
+            case ATTRIBUTE_NAME:
+            case ATTRIBUTE_DESCRIPTION:
+            case ATTRIBUTE_AFFILIATION:
+                setStringAttribute(attributeName, attributeValue);
+                break;
+            case ATTRIBUTE_FILE_PIDS_ENABLED:
+                setBooleanAttributeForFilePIDs(ctxt);
+                break;
+            default:
+                throw new IllegalCommandException("'" + attributeName + "' is not a supported attribute", this);
+        }
+
+        return ctxt.engine().submit(new UpdateDataverseCommand(dataverse, null, null, getRequest(), null));
+    }
+
+    /**
+     * Helper method to set a string attribute.
+     *
+     * @param attributeName  The name of the attribute.
+     * @param attributeValue The value of the attribute (must be a String).
+     * @throws IllegalCommandException if the provided attribute value is not of String type.
+     */
+    private void setStringAttribute(String attributeName, Object attributeValue) throws IllegalCommandException {
+        if (!(attributeValue instanceof String stringValue)) {
+            throw new IllegalCommandException("'" + attributeName + "' requires a string value", this);
+        }
+
+        switch (attributeName) {
+            case ATTRIBUTE_ALIAS:
+                dataverse.setAlias(stringValue);
+                break;
+            case ATTRIBUTE_NAME:
+                dataverse.setName(stringValue);
+                break;
+            case ATTRIBUTE_DESCRIPTION:
+                dataverse.setDescription(stringValue);
+                break;
+            case ATTRIBUTE_AFFILIATION:
+                dataverse.setAffiliation(stringValue);
+                break;
+            default:
+                throw new IllegalCommandException("Unsupported string attribute: " + attributeName, this);
+        }
+    }
+
+    /**
+     * Helper method to handle the "filePIDsEnabled" boolean attribute.
+     *
+     * @param ctxt The command context.
+     * @throws PermissionException if the user doesn't have permission to modify this attribute.
+     */
+    private void setBooleanAttributeForFilePIDs(CommandContext ctxt) throws CommandException {
+        if (!getRequest().getUser().isSuperuser()) {
+            throw new PermissionException("You must be a superuser to change this setting",
+                    this, Collections.singleton(Permission.EditDataset), dataverse);
+        }
+        if (!ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
+            throw new PermissionException("Changing File PID policy per collection is not enabled on this server",
+                    this, Collections.singleton(Permission.EditDataset), dataverse);
+        }
+
+        if (!(attributeValue instanceof Boolean)) {
+            throw new IllegalCommandException("'" + ATTRIBUTE_FILE_PIDS_ENABLED + "' requires a boolean value", this);
+        }
+
+        dataverse.setFilePIDsEnabled((Boolean) attributeValue);
+    }
+}

From 45cab7502a264108802c5462e46b48a27f118459 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 23 Oct 2024 14:38:33 +0100
Subject: [PATCH 097/137] Changed: recovered response message in
 updateAttribute endpoint

---
 src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 110adb60577..2be6b1e51c2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -624,7 +624,7 @@ public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam
             Dataverse dataverse = findDataverseOrDie(identifier);
             Object formattedValue = formatAttributeValue(attribute, value);
             dataverse = execCommand(new UpdateDataverseAttributeCommand(createDataverseRequest(getRequestUser(crc)), dataverse, attribute, formattedValue));
-            return ok(JsonPrinter.json(dataverse));
+            return ok("Update successful", JsonPrinter.json(dataverse));
         } catch (WrappedResponse e) {
             return e.getResponse();
         }

From e98ae05da45d2d4680807c257c6b95bfda81b8b5 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 23 Oct 2024 14:39:00 +0100
Subject: [PATCH 098/137] Added: IT cases for testAttributesApi

---
 .../iq/dataverse/api/DataversesIT.java        | 75 ++++++++++++++-----
 1 file changed, 57 insertions(+), 18 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 8c6a8244af1..6a040f27786 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -819,10 +819,9 @@ public void testImport() throws IOException, InterruptedException {
         Response deleteUserResponse = UtilIT.deleteUser(username);
         assertEquals(200, deleteUserResponse.getStatusCode());
     }
-    
-    @Test
-    public void testAttributesApi() throws Exception {
 
+    @Test
+    public void testAttributesApi() {
         Response createUser = UtilIT.createRandomUser();
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
 
@@ -837,30 +836,70 @@ public void testAttributesApi() throws Exception {
 
         String collectionAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
         String newCollectionAlias = collectionAlias + "RENAMED";
-        
-        // Change the alias of the collection: 
-        
-        Response changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "alias", newCollectionAlias, apiToken);
-        changeAttributeResp.prettyPrint();
-        
+
+        // Change the name of the collection:
+
+        String newCollectionName = "Renamed Name";
+        Response changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "name", newCollectionName, apiToken);
         changeAttributeResp.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("message.message", equalTo("Update successful"));
-        
-        // Check on the collection, under the new alias: 
-        
+
+        // Change the description of the collection:
+
+        String newDescription = "Renamed Description";
+        changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "description", newDescription, apiToken);
+        changeAttributeResp.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("message.message", equalTo("Update successful"));
+
+        // Change the affiliation of the collection:
+
+        String newAffiliation = "Renamed Affiliation";
+        changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "affiliation", newAffiliation, apiToken);
+        changeAttributeResp.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("message.message", equalTo("Update successful"));
+
+        // Cannot update filePIDsEnabled from a regular user:
+
+        changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "filePIDsEnabled", "true", apiToken);
+        changeAttributeResp.then().assertThat()
+                .statusCode(UNAUTHORIZED.getStatusCode());
+
+        // Change the alias of the collection:
+
+        changeAttributeResp = UtilIT.setCollectionAttribute(collectionAlias, "alias", newCollectionAlias, apiToken);
+        changeAttributeResp.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("message.message", equalTo("Update successful"));
+
+        // Check on the collection, under the new alias:
+
         Response collectionInfoResponse = UtilIT.exportDataverse(newCollectionAlias, apiToken);
-        collectionInfoResponse.prettyPrint();
-        
         collectionInfoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
-                .body("data.alias", equalTo(newCollectionAlias));
-        
+                .body("data.alias", equalTo(newCollectionAlias))
+                .body("data.name", equalTo(newCollectionName))
+                .body("data.description", equalTo(newDescription))
+                .body("data.affiliation", equalTo(newAffiliation));
+
         // Delete the collection (again, using its new alias):
-        
+
         Response deleteCollectionResponse = UtilIT.deleteDataverse(newCollectionAlias, apiToken);
-        deleteCollectionResponse.prettyPrint();
         assertEquals(OK.getStatusCode(), deleteCollectionResponse.getStatusCode());
+
+        // Cannot update root collection from a regular user:
+
+        changeAttributeResp = UtilIT.setCollectionAttribute("root", "name", newCollectionName, apiToken);
+        changeAttributeResp.then().assertThat()
+                .statusCode(UNAUTHORIZED.getStatusCode());
+
+        collectionInfoResponse = UtilIT.exportDataverse("root", apiToken);
+
+        collectionInfoResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.name", equalTo("Root"));
     }
 
     @Test

From 0453a85132a174a763a0f7367b65ece8ee9064d0 Mon Sep 17 00:00:00 2001
From: DieuwertjeBloemen <92870663+DieuwertjeBloemen@users.noreply.github.com>
Date: Thu, 24 Oct 2024 11:12:14 +0200
Subject: [PATCH 099/137] Update text.rst

Update link to metadata text guidelines in Dataverse style guide
---
 doc/sphinx-guides/source/style/text.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/style/text.rst b/doc/sphinx-guides/source/style/text.rst
index 4fb2352300c..10fbd08da4a 100644
--- a/doc/sphinx-guides/source/style/text.rst
+++ b/doc/sphinx-guides/source/style/text.rst
@@ -9,4 +9,4 @@ Here we describe the guidelines that help us provide helpful, clear and consiste
 Metadata Text Guidelines
 ========================
 
-These guidelines are maintained in `a Google Doc <https://docs.google.com/document/d/1uRk_dAZlaCS91YFbqE6L9Jwhwum7mOadkJ59XUx40Sg>`__ as we expect to make frequent changes to them. We welcome comments in the Google Doc.
\ No newline at end of file
+These guidelines are maintained in `a Google Doc <https://docs.google.com/document/d/1tY5t3gjrIgAGoRxVMWQSCh46fnbSmnFDLQ7aLkNLhJ8/>`__ as we expect to make frequent changes to them. We welcome comments in the Google Doc.

From 0b5f9a834b01dff526dbe76c250a5707a04a4656 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 24 Oct 2024 15:46:59 +0100
Subject: [PATCH 100/137] Changed: handling properties update through a DTO
 object for updateDataverse endpoint

---
 .../harvard/iq/dataverse/api/Dataverses.java  | 42 +++++---
 .../iq/dataverse/api/dto/DataverseDTO.java    | 63 ++++++++++++
 .../command/impl/UpdateDataverseCommand.java  | 42 +++++++-
 .../iq/dataverse/util/json/JsonParser.java    | 99 ++++++++++---------
 .../iq/dataverse/api/DataversesIT.java        | 20 ++++
 .../dataverse/util/json/JsonParserTest.java   | 20 ++--
 6 files changed, 204 insertions(+), 82 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 895d073bb47..25176b85689 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -3,12 +3,9 @@
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
 import edu.harvard.iq.dataverse.api.datadeposit.SwordServiceBean;
-import edu.harvard.iq.dataverse.api.dto.DataverseMetadataBlockFacetDTO;
+import edu.harvard.iq.dataverse.api.dto.*;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 
-import edu.harvard.iq.dataverse.api.dto.ExplicitGroupDTO;
-import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO;
-import edu.harvard.iq.dataverse.api.dto.RoleDTO;
 import edu.harvard.iq.dataverse.api.imports.ImportException;
 import edu.harvard.iq.dataverse.api.imports.ImportServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
@@ -128,7 +125,7 @@ public Response addRoot(@Context ContainerRequestContext crc, String body) {
     public Response addDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String parentIdtf) {
         Dataverse newDataverse;
         try {
-            newDataverse = parseAndValidateDataverseRequestBody(body, null);
+            newDataverse = parseAndValidateAddDataverseRequestBody(body);
         } catch (JsonParsingException jpe) {
             return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
         } catch (JsonParseException ex) {
@@ -159,20 +156,33 @@ public Response addDataverse(@Context ContainerRequestContext crc, String body,
         }
     }
 
+    private Dataverse parseAndValidateAddDataverseRequestBody(String body) throws JsonParsingException, JsonParseException {
+        try {
+            JsonObject addDataverseJson = JsonUtil.getJsonObject(body);
+            return jsonParser().parseDataverse(addDataverseJson);
+        } catch (JsonParsingException jpe) {
+            logger.log(Level.SEVERE, "Json: {0}", body);
+            throw jpe;
+        } catch (JsonParseException ex) {
+            logger.log(Level.SEVERE, "Error parsing dataverse from json: " + ex.getMessage(), ex);
+            throw ex;
+        }
+    }
+
     @PUT
     @AuthRequired
     @Path("{identifier}")
     public Response updateDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String identifier) {
-        Dataverse originalDataverse;
+        Dataverse dataverse;
         try {
-            originalDataverse = findDataverseOrDie(identifier);
+            dataverse = findDataverseOrDie(identifier);
         } catch (WrappedResponse e) {
             return e.getResponse();
         }
 
-        Dataverse updatedDataverse;
+        DataverseDTO updatedDataverseDTO;
         try {
-            updatedDataverse = parseAndValidateDataverseRequestBody(body, originalDataverse);
+            updatedDataverseDTO = parseAndValidateUpdateDataverseRequestBody(body);
         } catch (JsonParsingException jpe) {
             return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
         } catch (JsonParseException ex) {
@@ -180,13 +190,13 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
         }
 
         try {
-            List<DataverseFieldTypeInputLevel> inputLevels = parseInputLevels(body, originalDataverse);
+            List<DataverseFieldTypeInputLevel> inputLevels = parseInputLevels(body, dataverse);
             List<MetadataBlock> metadataBlocks = parseMetadataBlocks(body);
             List<DatasetFieldType> facets = parseFacets(body);
 
             AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
-            updatedDataverse = execCommand(new UpdateDataverseCommand(updatedDataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks));
-            return ok(json(updatedDataverse));
+            dataverse = execCommand(new UpdateDataverseCommand(dataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks, updatedDataverseDTO));
+            return ok(json(dataverse));
 
         } catch (WrappedResponse ww) {
             return handleWrappedResponse(ww);
@@ -198,15 +208,15 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod
         }
     }
 
-    private Dataverse parseAndValidateDataverseRequestBody(String body, Dataverse dataverseToUpdate) throws JsonParsingException, JsonParseException {
+    private DataverseDTO parseAndValidateUpdateDataverseRequestBody(String body) throws JsonParsingException, JsonParseException {
         try {
-            JsonObject dataverseJson = JsonUtil.getJsonObject(body);
-            return dataverseToUpdate != null ? jsonParser().parseDataverseUpdates(dataverseJson, dataverseToUpdate) : jsonParser().parseDataverse(dataverseJson);
+            JsonObject updateDataverseJson = JsonUtil.getJsonObject(body);
+            return jsonParser().parseDataverseDTO(updateDataverseJson);
         } catch (JsonParsingException jpe) {
             logger.log(Level.SEVERE, "Json: {0}", body);
             throw jpe;
         } catch (JsonParseException ex) {
-            logger.log(Level.SEVERE, "Error parsing dataverse from json: " + ex.getMessage(), ex);
+            logger.log(Level.SEVERE, "Error parsing DataverseDTO from json: " + ex.getMessage(), ex);
             throw ex;
         }
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java
new file mode 100644
index 00000000000..4f2f1032c07
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java
@@ -0,0 +1,63 @@
+package edu.harvard.iq.dataverse.api.dto;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseContact;
+
+import java.util.List;
+
+public class DataverseDTO {
+    private String alias;
+    private String name;
+    private String description;
+    private String affiliation;
+    private List<DataverseContact> dataverseContacts;
+    private Dataverse.DataverseType dataverseType;
+
+    public String getAlias() {
+        return alias;
+    }
+
+    public void setAlias(String alias) {
+        this.alias = alias;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public String getAffiliation() {
+        return affiliation;
+    }
+
+    public void setAffiliation(String affiliation) {
+        this.affiliation = affiliation;
+    }
+
+    public List<DataverseContact> getDataverseContacts() {
+        return dataverseContacts;
+    }
+
+    public void setDataverseContacts(List<DataverseContact> dataverseContacts) {
+        this.dataverseContacts = dataverseContacts;
+    }
+
+    public Dataverse.DataverseType getDataverseType() {
+        return dataverseType;
+    }
+
+    public void setDataverseType(Dataverse.DataverseType dataverseType) {
+        this.dataverseType = dataverseType;
+    }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index 14d9e408be8..55cc3708097 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.Dataverse.DataverseType;
+import edu.harvard.iq.dataverse.api.dto.DataverseDTO;
 import edu.harvard.iq.dataverse.authorization.Permission;
 
 import static edu.harvard.iq.dataverse.dataverse.DataverseUtil.validateDataverseMetadataExternally;
@@ -22,29 +23,32 @@
 @RequiredPermissions(Permission.EditDataverse)
 public class UpdateDataverseCommand extends AbstractWriteDataverseCommand {
     private final List<Dataverse> featuredDataverseList;
+    private final DataverseDTO updatedDataverseDTO;
 
     private boolean datasetsReindexRequired = false;
 
-    public UpdateDataverseCommand(Dataverse editedDv,
+    public UpdateDataverseCommand(Dataverse dataverse,
                                   List<DatasetFieldType> facets,
                                   List<Dataverse> featuredDataverses,
                                   DataverseRequest request,
                                   List<DataverseFieldTypeInputLevel> inputLevels) {
-        this(editedDv, facets, featuredDataverses, request, inputLevels, null);
+        this(dataverse, facets, featuredDataverses, request, inputLevels, null, null);
     }
 
-    public UpdateDataverseCommand(Dataverse editedDv,
+    public UpdateDataverseCommand(Dataverse dataverse,
                                   List<DatasetFieldType> facets,
                                   List<Dataverse> featuredDataverses,
                                   DataverseRequest request,
                                   List<DataverseFieldTypeInputLevel> inputLevels,
-                                  List<MetadataBlock> metadataBlocks) {
-        super(editedDv, editedDv, request, facets, inputLevels, metadataBlocks);
+                                  List<MetadataBlock> metadataBlocks,
+                                  DataverseDTO updatedDataverseDTO) {
+        super(dataverse, dataverse, request, facets, inputLevels, metadataBlocks);
         if (featuredDataverses != null) {
             this.featuredDataverseList = new ArrayList<>(featuredDataverses);
         } else {
             this.featuredDataverseList = null;
         }
+        this.updatedDataverseDTO = updatedDataverseDTO;
     }
 
     @Override
@@ -87,9 +91,37 @@ protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandExcep
             }
         }
 
+        if (updatedDataverseDTO != null) {
+            updateDataverseFromDTO(dataverse, updatedDataverseDTO);
+        }
+
         return dataverse;
     }
 
+    private void updateDataverseFromDTO(Dataverse dataverse, DataverseDTO dto) {
+        if (dto.getAlias() != null) {
+            dataverse.setAlias(dto.getAlias());
+        }
+        if (dto.getName() != null) {
+            dataverse.setName(dto.getName());
+        }
+        if (dto.getDescription() != null) {
+            dataverse.setDescription(dto.getDescription());
+        }
+        if (dto.getAffiliation() != null) {
+            dataverse.setAffiliation(dto.getAffiliation());
+        }
+        if (dto.getDataverseContacts() != null) {
+            dataverse.setDataverseContacts(dto.getDataverseContacts());
+            for (DataverseContact dc : dataverse.getDataverseContacts()) {
+                dc.setDataverse(dataverse);
+            }
+        }
+        if (dto.getDataverseType() != null) {
+            dataverse.setDataverseType(dto.getDataverseType());
+        }
+    }
+
     @Override
     public boolean onSuccess(CommandContext ctxt, Object r) {
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index f63e4c4fd9c..8552389525d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -19,6 +19,7 @@
 import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
 import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
 import edu.harvard.iq.dataverse.api.Util;
+import edu.harvard.iq.dataverse.api.dto.DataverseDTO;
 import edu.harvard.iq.dataverse.api.dto.FieldDTO;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
@@ -48,6 +49,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -130,8 +132,19 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
         dv.setFacetRoot(jobj.getBoolean("facetRoot", false));
         dv.setAffiliation(jobj.getString("affiliation", null));
 
-        updateDataverseContacts(dv, jobj);
-        
+        if (jobj.containsKey("dataverseContacts")) {
+            JsonArray dvContacts = jobj.getJsonArray("dataverseContacts");
+            int i = 0;
+            List<DataverseContact> dvContactList = new LinkedList<>();
+            for (JsonValue jsv : dvContacts) {
+                DataverseContact dvc = new DataverseContact(dv);
+                dvc.setContactEmail(getMandatoryString((JsonObject) jsv, "contactEmail"));
+                dvc.setDisplayOrder(i++);
+                dvContactList.add(dvc);
+            }
+            dv.setDataverseContacts(dvContactList);
+        }
+
         if (jobj.containsKey("theme")) {
             DataverseTheme theme = parseDataverseTheme(jobj.getJsonObject("theme"));
             dv.setDataverseTheme(theme);
@@ -139,7 +152,13 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
         }
 
         dv.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED); // default
-        updateDataverseType(dv, jobj);
+        String receivedDataverseType = jobj.getString("dataverseType", null);
+        if (receivedDataverseType != null) {
+            Arrays.stream(Dataverse.DataverseType.values())
+                    .filter(type -> type.name().equals(receivedDataverseType))
+                    .findFirst()
+                    .ifPresent(dv::setDataverseType);
+        }
 
         if (jobj.containsKey("filePIDsEnabled")) {
             dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled"));
@@ -147,7 +166,7 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
 
         /*  We decided that subject is not user set, but gotten from the subject of the dataverse's
             datasets - leavig this code in for now, in case we need to go back to it at some point
-        
+
         if (jobj.containsKey("dataverseSubjects")) {
             List<ControlledVocabularyValue> dvSubjectList = new LinkedList<>();
             DatasetFieldType subjectType = datasetFieldSvc.findByName(DatasetFieldConstant.subject);
@@ -170,63 +189,49 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
             dv.setDataverseSubjects(dvSubjectList);
         }
         */
-                
+
         return dv;
     }
 
-    public Dataverse parseDataverseUpdates(JsonObject jsonObject, Dataverse dataverseToUpdate) throws JsonParseException {
-        String alias = jsonObject.getString("alias", null);
-        if (alias != null) {
-            dataverseToUpdate.setAlias(alias);
-        }
-
-        String name = jsonObject.getString("name", null);
-        if (name != null) {
-            dataverseToUpdate.setName(name);
-        }
-
-        String description = jsonObject.getString("description", null);
-        if (description != null) {
-            dataverseToUpdate.setDescription(description);
-        }
-
-        String affiliation = jsonObject.getString("affiliation", null);
-        if (affiliation != null) {
-            dataverseToUpdate.setAffiliation(affiliation);
-        }
-
-        updateDataverseType(dataverseToUpdate, jsonObject);
+    public DataverseDTO parseDataverseDTO(JsonObject jsonObject) throws JsonParseException {
+        DataverseDTO dataverseDTO = new DataverseDTO();
 
-        updateDataverseContacts(dataverseToUpdate, jsonObject);
+        setDataverseDTOPropertyIfPresent(jsonObject, "alias", dataverseDTO::setAlias);
+        setDataverseDTOPropertyIfPresent(jsonObject, "name", dataverseDTO::setName);
+        setDataverseDTOPropertyIfPresent(jsonObject, "description", dataverseDTO::setDescription);
+        setDataverseDTOPropertyIfPresent(jsonObject, "affiliation", dataverseDTO::setAffiliation);
 
-        return dataverseToUpdate;
-    }
-
-    private void updateDataverseType(Dataverse dataverse, JsonObject jsonObject) {
-        String receivedDataverseType = jsonObject.getString("dataverseType", null);
-        if (receivedDataverseType != null) {
+        String dataverseType = jsonObject.getString("dataverseType", null);
+        if (dataverseType != null) {
             Arrays.stream(Dataverse.DataverseType.values())
-                    .filter(type -> type.name().equals(receivedDataverseType))
+                    .filter(type -> type.name().equals(dataverseType))
                     .findFirst()
-                    .ifPresent(dataverse::setDataverseType);
+                    .ifPresent(dataverseDTO::setDataverseType);
         }
-    }
 
-    private void updateDataverseContacts(Dataverse dataverse, JsonObject jsonObject) throws JsonParseException {
         if (jsonObject.containsKey("dataverseContacts")) {
             JsonArray dvContacts = jsonObject.getJsonArray("dataverseContacts");
-            int i = 0;
-            List<DataverseContact> dvContactList = new LinkedList<>();
-            for (JsonValue jsv : dvContacts) {
-                DataverseContact dvc = new DataverseContact(dataverse);
-                dvc.setContactEmail(getMandatoryString((JsonObject) jsv, "contactEmail"));
-                dvc.setDisplayOrder(i++);
-                dvContactList.add(dvc);
+            List<DataverseContact> contacts = new ArrayList<>();
+            for (int i = 0; i < dvContacts.size(); i++) {
+                JsonObject contactObj = dvContacts.getJsonObject(i);
+                DataverseContact contact = new DataverseContact();
+                contact.setContactEmail(getMandatoryString(contactObj, "contactEmail"));
+                contact.setDisplayOrder(i);
+                contacts.add(contact);
             }
-            dataverse.setDataverseContacts(dvContactList);
+            dataverseDTO.setDataverseContacts(contacts);
         }
+
+        return dataverseDTO;
     }
-    
+
+    private void setDataverseDTOPropertyIfPresent(JsonObject jsonObject, String key, Consumer<String> setter) {
+        String value = jsonObject.getString(key, null);
+        if (value != null) {
+            setter.accept(value);
+        }
+    }
+
     public DataverseTheme parseDataverseTheme(JsonObject obj) {
 
         DataverseTheme theme = new DataverseTheme();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index c311fa1016e..e6ec3cf5400 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -1341,6 +1341,26 @@ public void testUpdateDataverse() {
                 apiToken
         );
         updateDataverseResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // User with unprivileged API token cannot update Root dataverse
+        updateDataverseResponse = UtilIT.updateDataverse(
+                "root",
+                newAlias,
+                newName,
+                newAffiliation,
+                newDataverseType,
+                newContactEmails,
+                newInputLevelNames,
+                newFacetIds,
+                newMetadataBlockNames,
+                apiToken
+        );
+        updateDataverseResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode());
+
+        Response rootCollectionInfoResponse = UtilIT.exportDataverse("root", apiToken);
+        rootCollectionInfoResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.name", equalTo("Root"));
     }
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
index 2cffa7d921c..f241a5d1dda 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
@@ -8,6 +8,7 @@
 import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
 import edu.harvard.iq.dataverse.DataverseTheme.Alignment;
 import edu.harvard.iq.dataverse.UserNotification.Type;
+import edu.harvard.iq.dataverse.api.dto.DataverseDTO;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroupProvider;
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
@@ -266,32 +267,23 @@ public void testParseCompleteDataverse() throws JsonParseException {
     }
 
     /**
-     * Test that a JSON object passed for a complete Dataverse update is correctly parsed.
-     * This checks that all properties are parsed into the correct dataverse properties.
+     * Test that a JSON object passed for a DataverseDTO is correctly parsed.
+     * This checks that all properties are parsed into the correct DataverseDTO properties.
      * @throws JsonParseException when this test is broken.
      */
     @Test
-    public void parseDataverseUpdates() throws JsonParseException {
-        Dataverse dataverse = new Dataverse();
-            dataverse.setName("Name to update");
-            dataverse.setAlias("aliasToUpdate");
-            dataverse.setAffiliation("Affiliation to update");
-            dataverse.setDescription("Description to update");
-            dataverse.setDataverseType(Dataverse.DataverseType.DEPARTMENT);
-            List<DataverseContact> originalContacts = new ArrayList<>();
-            originalContacts.add(new DataverseContact(dataverse, "updatethis@example.edu"));
-            dataverse.setDataverseContacts(originalContacts);
+    public void parseDataverseDTO() throws JsonParseException {
         JsonObject dvJson;
         try (FileReader reader = new FileReader("doc/sphinx-guides/source/_static/api/dataverse-complete.json")) {
             dvJson = Json.createReader(reader).readObject();
-            Dataverse actual = sut.parseDataverseUpdates(dvJson, dataverse);
+            DataverseDTO actual = sut.parseDataverseDTO(dvJson);
             assertEquals("Scientific Research", actual.getName());
             assertEquals("science", actual.getAlias());
             assertEquals("Scientific Research University", actual.getAffiliation());
             assertEquals("We do all the science.", actual.getDescription());
             assertEquals("LABORATORY", actual.getDataverseType().toString());
             assertEquals(2, actual.getDataverseContacts().size());
-            assertEquals("pi@example.edu,student@example.edu", actual.getContactEmails());
+            assertEquals("pi@example.edu,student@example.edu", actual.getDataverseContacts().get(0).getContactEmail());
             assertEquals(0, actual.getDataverseContacts().get(0).getDisplayOrder());
             assertEquals(1, actual.getDataverseContacts().get(1).getDisplayOrder());
         } catch (IOException ioe) {

From 224cc963ddcaa5feb5d8ad931b6152e6daa0f201 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 24 Oct 2024 11:17:31 -0400
Subject: [PATCH 101/137] Fix for timing issue in test

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java
index e8426b638d7..a0b9f5325d0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java
@@ -94,7 +94,8 @@ public void testCreateSoftwareDatasetNative() {
         String dataset2Pid = JsonPath.from(createDataset.getBody().asString()).getString("data.persistentId");
 
         UtilIT.publishDatasetViaNativeApi(dataset2Pid, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
-
+        //An explicit sleep is needed here because the searchAndShowFacets won't sleep for the query used here
+        UtilIT.sleepForReindex(dataset2Pid, apiToken, 5);
         Response searchCollection = UtilIT.searchAndShowFacets("parentName:" + dataverseAlias, null);
         searchCollection.prettyPrint();
         searchCollection.then().assertThat()

From 1012763530091fb7f0cd04c7b014d77d2ff4b9b7 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 24 Oct 2024 15:06:28 -0400
Subject: [PATCH 102/137] remove link to empty doc of tips #10962

---
 doc/sphinx-guides/source/admin/metadatacustomization.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index e5326efebef..2a5cb1fbbc8 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -644,12 +644,6 @@ Tips from the Dataverse Community
 
 When creating new metadata blocks, please review the :doc:`/style/text` section of the Style Guide, which includes guidance about naming metadata fields and writing text for metadata tooltips and watermarks.
 
-If there are tips that you feel are omitted from this document, please open an issue at https://github.com/IQSS/dataverse/issues and consider making a pull request to make improvements. You can find this document at https://github.com/IQSS/dataverse/blob/develop/doc/sphinx-guides/source/admin/metadatacustomization.rst
-
-Alternatively, you are welcome to request "edit" access to this "Tips for Dataverse Software metadata blocks from the community" Google doc: https://docs.google.com/document/d/1XpblRw0v0SvV-Bq6njlN96WyHJ7tqG0WWejqBdl7hE0/edit?usp=sharing
-
-The thinking is that the tips can become issues and the issues can eventually be worked on as features to improve the Dataverse Software metadata system.
-
 Development Tasks Specific to Changing Fields in Core Metadata Blocks
 ---------------------------------------------------------------------
 

From f0f22b971e40bfb5e0c68de697463363eafd433f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 24 Oct 2024 19:55:53 +0000
Subject: [PATCH 103/137] Bump org.apache.james:apache-mime4j-core from 0.8.7
 to 0.8.10

Bumps org.apache.james:apache-mime4j-core from 0.8.7 to 0.8.10.

---
updated-dependencies:
- dependency-name: org.apache.james:apache-mime4j-core
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 1e70ff68480..b2344989569 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,7 +72,7 @@
             <dependency>
                 <groupId>org.apache.james</groupId>
                 <artifactId>apache-mime4j-core</artifactId>
-                <version>0.8.7</version>
+                <version>0.8.10</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.james</groupId>

From 473286c76b5735ae48aac694a154b3591f125214 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 25 Oct 2024 09:57:17 -0400
Subject: [PATCH 104/137] fix blank query, emphasize Guest user in comment

---
 .../edu/harvard/iq/dataverse/search/SearchServiceBean.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 245f778da23..6fadd9a04a3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -332,7 +332,7 @@ public SolrQueryResponse search(
         // PERMISSION FILTER QUERY
         // -----------------------------------
         String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, onlyDatatRelatedToMe, addFacets);
-        if (permissionFilterQuery != null) {
+        if (!StringUtils.isBlank(permissionFilterQuery)) {
             solrQuery.addFilterQuery(permissionFilterQuery);
         }
         
@@ -1047,7 +1047,7 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
         // add joins on all the non-public groups that may exist for the
         // user:
 
-        // Authenticated users and GuestUser may be part of one or more groups; such
+        // Authenticated users - and GuestUser - may be part of one or more groups; such
         // as IP Groups.
         groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
 

From 940b3fb3f99d587238c41b141d7299f8aefc2bd7 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 25 Oct 2024 10:22:01 -0400
Subject: [PATCH 105/137] alternate way of emphasizing

---
 .../java/edu/harvard/iq/dataverse/search/SearchServiceBean.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 6fadd9a04a3..de75c88009f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -1047,7 +1047,7 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
         // add joins on all the non-public groups that may exist for the
         // user:
 
-        // Authenticated users - and GuestUser - may be part of one or more groups; such
+        // Authenticated users, *and the GuestUser*, may be part of one or more groups; such
         // as IP Groups.
         groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
 

From f851163df6a17106599f675e205dd5985ca68c94 Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Fri, 25 Oct 2024 16:55:54 +0200
Subject: [PATCH 106/137] Ordering subfields while displaying dataset version
 differences

---
 .../iq/dataverse/DatasetVersionDifference.java     | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java
index eca0c84ae84..c32f49e985e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java
@@ -152,8 +152,7 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin
         getReplacedFiles();
         initDatasetFilesDifferencesList();
 
-        //Sort within blocks by datasetfieldtype dispaly order then....
-        //sort via metadatablock order - citation first...
+        //Sort within blocks by datasetfieldtype display order
         for (List<DatasetField[]> blockList : detailDataByBlock) {
             Collections.sort(blockList, (DatasetField[] l1, DatasetField[] l2) -> {
                     DatasetField dsfa = l1[0];  //(DatasetField[]) l1.get(0);
@@ -163,6 +162,17 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin
                 return Integer.valueOf(a).compareTo(b);
             });
         }
+        //Sort existing compoundValues by datasetfieldtype display order
+        for (List<DatasetField[]> blockList : detailDataByBlock) {
+            for (DatasetField[] dfarr : blockList) {
+                for (DatasetField df : dfarr) {
+                    for (DatasetFieldCompoundValue dfcv : df.getDatasetFieldCompoundValues()) {
+                        Collections.sort(dfcv.getChildDatasetFields(), DatasetField.DisplayOrder);
+                    }
+                }
+            }
+        }
+        //Sort via metadatablock order
         Collections.sort(detailDataByBlock, (List l1, List l2) -> {
                 DatasetField dsfa[] = (DatasetField[]) l1.get(0);
                 DatasetField dsfb[] = (DatasetField[]) l2.get(0);

From b3147a083d140b6a9a2feea21aa043621f692fcc Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Fri, 25 Oct 2024 17:04:38 +0200
Subject: [PATCH 107/137] adding release note

---
 doc/release-notes/10969-order-subfields-version-difference.md | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 doc/release-notes/10969-order-subfields-version-difference.md

diff --git a/doc/release-notes/10969-order-subfields-version-difference.md b/doc/release-notes/10969-order-subfields-version-difference.md
new file mode 100644
index 00000000000..3f245ebe069
--- /dev/null
+++ b/doc/release-notes/10969-order-subfields-version-difference.md
@@ -0,0 +1,2 @@
+Bug Fix:
+In order to facilitate the comparison between the draft version and the published version of a dataset, a sort on subfields has been added (#10969)
\ No newline at end of file

From e76f2ff40a5aadd352bfef116f0f0db393b922cf Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 25 Oct 2024 14:46:07 -0400
Subject: [PATCH 108/137] Revert "remove link to empty doc of tips #10962"

This reverts commit 1012763530091fb7f0cd04c7b014d77d2ff4b9b7.
---
 doc/sphinx-guides/source/admin/metadatacustomization.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index 2a5cb1fbbc8..e5326efebef 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -644,6 +644,12 @@ Tips from the Dataverse Community
 
 When creating new metadata blocks, please review the :doc:`/style/text` section of the Style Guide, which includes guidance about naming metadata fields and writing text for metadata tooltips and watermarks.
 
+If there are tips that you feel are omitted from this document, please open an issue at https://github.com/IQSS/dataverse/issues and consider making a pull request to make improvements. You can find this document at https://github.com/IQSS/dataverse/blob/develop/doc/sphinx-guides/source/admin/metadatacustomization.rst
+
+Alternatively, you are welcome to request "edit" access to this "Tips for Dataverse Software metadata blocks from the community" Google doc: https://docs.google.com/document/d/1XpblRw0v0SvV-Bq6njlN96WyHJ7tqG0WWejqBdl7hE0/edit?usp=sharing
+
+The thinking is that the tips can become issues and the issues can eventually be worked on as features to improve the Dataverse Software metadata system.
+
 Development Tasks Specific to Changing Fields in Core Metadata Blocks
 ---------------------------------------------------------------------
 

From 04f7f768fa5df05fa61e4387f08f56b88316dbc4 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 28 Oct 2024 10:00:30 -0400
Subject: [PATCH 109/137] fix get major version when deaccessioned

---
 .../edu/harvard/iq/dataverse/Dataset.java     | 11 +++++-
 .../edu/harvard/iq/dataverse/DatasetTest.java | 39 ++++++++++++++++++-
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 40ed491a302..78579b1de21 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -483,8 +483,17 @@ public Date getMostRecentMajorVersionReleaseDate() {
         if (this.isHarvested()) {
             return getVersions().get(0).getReleaseTime();
         } else {
+            Long majorVersion = null;
             for (DatasetVersion version : this.getVersions()) {
-                if (version.isReleased() && version.getMinorVersionNumber().equals((long) 0)) {
+                if (version.isReleased()) {
+                    if (version.getMinorVersionNumber().equals((long) 0)) {
+                        return version.getReleaseTime();
+                    } else if (majorVersion == null) {
+                        majorVersion = version.getVersionNumber();
+                    }
+                } else if (version.isDeaccessioned() && majorVersion != null
+                        && majorVersion.longValue() == version.getVersionNumber().longValue()
+                        && version.getMinorVersionNumber().equals((long) 0)) {
                     return version.getReleaseTime();
                 }
             }
diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetTest.java
index 2153a336303..687e0af5b81 100644
--- a/src/test/java/edu/harvard/iq/dataverse/DatasetTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/DatasetTest.java
@@ -10,6 +10,7 @@
 import static org.junit.jupiter.api.Assertions.*;
 
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -240,5 +241,41 @@ public void datasetShouldBeDeaccessionedWithDeaccessionedAndDeaccessionedVersion
 
         assertTrue(dataset.isDeaccessioned());
     }
- 
+
+    @Test
+    public void testGetMostRecentMajorVersionReleaseDateWithDeaccessionedVersions() {
+        List<DatasetVersion> versionList = new ArrayList<DatasetVersion>();
+
+        long ver = 5;
+        // 5.2
+        DatasetVersion relVersion = new DatasetVersion();
+        relVersion.setVersionState(VersionState.RELEASED);
+        relVersion.setMinorVersionNumber(2L);
+        relVersion.setVersionNumber(ver);
+        versionList.add(relVersion);
+
+        // 5.1
+        relVersion = new DatasetVersion();
+        relVersion.setVersionState(VersionState.DEACCESSIONED);
+        relVersion.setMinorVersionNumber(1L);
+        relVersion.setVersionNumber(ver);
+        versionList.add(relVersion);
+
+        // 5.0, 4.0, 3.0, 2.0, 1.0
+        while  (ver > 0) {
+            DatasetVersion deaccessionedVersion = new DatasetVersion();
+            deaccessionedVersion.setVersionState(VersionState.DEACCESSIONED);
+            // only add an actual date to v5.0 so the assertNotNull will only pass if this version's date is returned
+            deaccessionedVersion.setReleaseTime((ver == 5) ? new Date() : null);
+            deaccessionedVersion.setMinorVersionNumber(0L);
+            deaccessionedVersion.setVersionNumber(ver--);
+            versionList.add(deaccessionedVersion);
+        }
+
+        Dataset dataset = new Dataset();
+        dataset.setVersions(versionList);
+
+        Date releaseDate = dataset.getMostRecentMajorVersionReleaseDate();
+        assertNotNull(releaseDate);
+    }
 }

From 7010eada5a48a858ee0517a30f64d22ca236ec49 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 28 Oct 2024 11:17:18 -0400
Subject: [PATCH 110/137] adding release note

---
 ...shed-files-appearing-in-search-results-for-anon-user.md | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md

diff --git a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
new file mode 100644
index 00000000000..6e6d575ddcf
--- /dev/null
+++ b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
@@ -0,0 +1,7 @@
+A bug fix was made that gets the major version of a Dataset when all major version were deaccessioned. This fixes the incorrect showing of the files as unpublished in the search list even when they are published.
+This fix affects the indexing meaning these datasets must be re-indexed once Dataverse is updated. This can be manually done by calling the index API for each affected Dataset.
+
+Example:
+```shell
+curl http://localhost:8080/api/admin/index/dataset?persistentId=doi:10.7910/DVN/6X4ZZL
+```

From 81672be40faa97a02498880d1866d30d5f17cf9e Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 28 Oct 2024 11:18:42 -0400
Subject: [PATCH 111/137] adding release note

---
 ...published-files-appearing-in-search-results-for-anon-user.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
index 6e6d575ddcf..a43e4ce0d36 100644
--- a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
+++ b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
@@ -1,4 +1,4 @@
-A bug fix was made that gets the major version of a Dataset when all major version were deaccessioned. This fixes the incorrect showing of the files as unpublished in the search list even when they are published.
+A bug fix was made that gets the major version of a Dataset when all major versions were deaccessioned. This fixes the incorrect showing of the files as "Unpublished" in the search list even when they are published.
 This fix affects the indexing meaning these datasets must be re-indexed once Dataverse is updated. This can be manually done by calling the index API for each affected Dataset.
 
 Example:

From f2a9b0aee0094c7460c219941fc40c0065cf3e95 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 28 Oct 2024 13:46:49 -0400
Subject: [PATCH 112/137] Update
 doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 ...published-files-appearing-in-search-results-for-anon-user.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
index a43e4ce0d36..126ef957447 100644
--- a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
+++ b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
@@ -1,3 +1,5 @@
+## Unpublished file bug fix
+
 A bug fix was made that gets the major version of a Dataset when all major versions were deaccessioned. This fixes the incorrect showing of the files as "Unpublished" in the search list even when they are published.
 This fix affects the indexing meaning these datasets must be re-indexed once Dataverse is updated. This can be manually done by calling the index API for each affected Dataset.
 

From c4c3e2849745a949675ea73d1b1a4ff8a1caadfb Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Mon, 28 Oct 2024 13:47:36 -0400
Subject: [PATCH 113/137] Update
 doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 ...published-files-appearing-in-search-results-for-anon-user.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
index 126ef957447..394f44419e4 100644
--- a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
+++ b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
@@ -1,7 +1,7 @@
 ## Unpublished file bug fix
 
 A bug fix was made that gets the major version of a Dataset when all major versions were deaccessioned. This fixes the incorrect showing of the files as "Unpublished" in the search list even when they are published.
-This fix affects the indexing meaning these datasets must be re-indexed once Dataverse is updated. This can be manually done by calling the index API for each affected Dataset.
+This fix affects the indexing, meaning these datasets must be re-indexed once Dataverse is updated. This can be manually done by calling the index API for each affected Dataset.
 
 Example:
 ```shell

From 2dfdaa27e3e2f7b2dcb21b2b5ca4f4a47cd245d5 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 28 Oct 2024 13:51:14 -0400
Subject: [PATCH 114/137] add #10947 and #10974 to release note

---
 ...published-files-appearing-in-search-results-for-anon-user.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
index 394f44419e4..66ea04b124f 100644
--- a/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
+++ b/doc/release-notes/10947-unpublished-files-appearing-in-search-results-for-anon-user.md
@@ -7,3 +7,5 @@ Example:
 ```shell
 curl http://localhost:8080/api/admin/index/dataset?persistentId=doi:10.7910/DVN/6X4ZZL
 ```
+
+See also #10947 and #10974.

From ad3505143e60e4b9cea896c52b3db782a77f2f07 Mon Sep 17 00:00:00 2001
From: Ludovic DANIEL <ludovic.daniel@smile.fr>
Date: Wed, 30 Oct 2024 17:39:50 +0100
Subject: [PATCH 115/137] add release note

---
 doc/release-notes/10772-fix-importDDI-otherId.md | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 doc/release-notes/10772-fix-importDDI-otherId.md

diff --git a/doc/release-notes/10772-fix-importDDI-otherId.md b/doc/release-notes/10772-fix-importDDI-otherId.md
new file mode 100644
index 00000000000..d5a9018b2b2
--- /dev/null
+++ b/doc/release-notes/10772-fix-importDDI-otherId.md
@@ -0,0 +1,2 @@
+Bug Fix :
+This PR fixes the `edu.harvard.iq.dataverse.util.json.JsonParseException: incorrect multiple   for field otherId` error when DDI harvested data contains multiple ortherId.
\ No newline at end of file

From 93dd4231fa2d7d180412c93435b6f8e682816bc0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 30 Oct 2024 13:19:30 -0400
Subject: [PATCH 116/137] add test for returnDatasetFieldTypes #10984

---
 .../java/edu/harvard/iq/dataverse/api/DataversesIT.java  | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 6a040f27786..0eb2670b272 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -911,6 +911,15 @@ public void testListMetadataBlocks() {
         createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
+        Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
+        listMetadataBlocks.prettyPrint();
+        listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+        listMetadataBlocks.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].name", is("citation"))
+                // failing? "fields" is empty, showing {}
+                .body("data[0].fields.title.displayOnCreate", equalTo(true));
+
         Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken);
         setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode());
 

From 51794df26c00ce5ab9eb1fdbeb7d5aa91e8576fc Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 30 Oct 2024 16:45:57 -0400
Subject: [PATCH 117/137] Add FIXME about inheritence. Add "if #10984 fixed" to
 tests.

---
 .../iq/dataverse/DatasetFieldServiceBean.java |  2 +-
 .../iq/dataverse/api/DataversesIT.java        | 29 ++++++++++++++-----
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
index 91150b79505..c977ae784bd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
@@ -947,7 +947,7 @@ public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock m
         criteriaQuery.where(
                 criteriaBuilder.equal(dataverseRoot.get("id"), dataverse.getId()), // Match the Dataverse ID.
                 criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID.
-                metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse.
+                metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse. FIXME: inherit blocks from parent
                 datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock.
                 criteriaBuilder.or(includedAsInputLevelPredicate, hasNoInputLevelPredicate), // Include DatasetFieldTypes based on the input level predicates.
                 displayedOnCreatePredicate // Apply the display-on-create filter if necessary.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 0eb2670b272..9e3555555e8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -911,14 +911,27 @@ public void testListMetadataBlocks() {
         createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
-        Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
-        listMetadataBlocks.prettyPrint();
-        listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
-        listMetadataBlocks.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].name", is("citation"))
-                // failing? "fields" is empty, showing {}
-                .body("data[0].fields.title.displayOnCreate", equalTo(true));
+        boolean issue10984fixed = false;
+        // See https://github.com/IQSS/dataverse/issues/10984
+        if (issue10984fixed) {
+            Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
+            listMetadataBlocks.prettyPrint();
+            listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+            listMetadataBlocks.then().assertThat()
+                    .statusCode(OK.getStatusCode())
+                    .body("data[0].name", is("citation"))
+                    .body("data[0].fields.title.displayOnCreate", equalTo(true));
+
+        } else {
+            Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
+            listMetadataBlocks.prettyPrint();
+            listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+            listMetadataBlocks.then().assertThat()
+                    .statusCode(OK.getStatusCode())
+                    .body("data[0].name", is("citation"))
+                    // "fields" should be more like 28, not 0
+                    .body("data[0].fields.size()", is(0));
+        }
 
         Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken);
         setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode());

From db181848cd9c94044222598a97a7cd891a16e8de Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 30 Oct 2024 17:46:56 -0400
Subject: [PATCH 118/137] fixes the validation method in harvesting import that
 got broken in 10836. #10989

---
 .../api/imports/ImportServiceBean.java        | 49 +++++++++++++------
 1 file changed, 33 insertions(+), 16 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index 66f48bfb872..b203738a9fd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -359,12 +359,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             if (harvestedVersion.getReleaseTime() == null) {
                 harvestedVersion.setReleaseTime(oaiDateStamp);
             }
-            
-            // is this the right place to call tidyUpFields()? 
-            // usually it is called within the body of the create/update commands
-            // later on.
-            DatasetFieldUtil.tidyUpFields(harvestedVersion.getDatasetFields(), true);
-            
+                        
             // Check data against validation constraints. 
             // Make an attempt to sanitize any invalid fields encountered - 
             // missing required fields or invalid values, by filling the values 
@@ -382,7 +377,9 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
             if (sanitized) {
                 validateVersionMetadata(harvestedVersion, cleanupLog);
             }
-            
+               
+            DatasetFieldUtil.tidyUpFields(harvestedVersion.getDatasetFields(), true);
+
             if (existingDataset != null) {
                 importedDataset = engineSvc.submit(new UpdateHarvestedDatasetCommand(existingDataset, harvestedVersion, dataverseRequest));
             } else {
@@ -742,15 +739,35 @@ private boolean validateVersionMetadata(DatasetVersion version, boolean sanitize
         boolean fixed = false;
         Set<ConstraintViolation> invalidViolations = version.validate();
         if (!invalidViolations.isEmpty()) {
-            for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
-                DatasetFieldValue f = v.getRootBean();
-
-                String msg = "Invalid metadata field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
-                        + "Invalid value:  '" + f.getValue() + "'";
-                if (sanitize) {
-                    msg += ", replaced with '" + DatasetField.NA_VALUE + "'";
-                    f.setValue(DatasetField.NA_VALUE);
-                    fixed = true;
+            for (ConstraintViolation v : invalidViolations) {
+                Object invalid = v.getRootBean();
+                String msg = "";
+                if (invalid instanceof DatasetField) {
+                    DatasetField f = (DatasetField) invalid; 
+                    
+                    msg += "Missing required field: " + f.getDatasetFieldType().getDisplayName() + ";";                  
+                    if (sanitize) {
+                        msg += " populated with '" + DatasetField.NA_VALUE + "'";
+                        f.setSingleValue(DatasetField.NA_VALUE);
+                        fixed = true;
+                    }
+                } else if (invalid instanceof DatasetFieldValue) {
+                    DatasetFieldValue fv = (DatasetFieldValue) invalid;
+                    
+                    msg += "Invalid metadata field: " + fv.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
+                            + "Invalid value:  '" + fv.getValue() + "'";
+                    if (sanitize) {
+                        msg += ", replaced with '" + DatasetField.NA_VALUE + "'";
+                        fv.setValue(DatasetField.NA_VALUE);
+                        fixed = true;
+                    }
+                } else {
+                    // DatasetVersion.validate() can also produce constraint violations
+                    // in TermsOfUse and FileMetadata classes. 
+                    // We do not make any attempt to sanitize those.
+                    if (invalid != null) {
+                        msg += "Invalid " + invalid.getClass().getName() + ": " + v.getMessage();
+                    }
                 }
                 cleanupLog.println(msg);
 

From ed8a889bdc0df615d2f49a66c67ecfa41559099b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 31 Oct 2024 12:49:26 +0000
Subject: [PATCH 119/137] Fixed: always querying the owner if the dataverse is
 not MetadataBlock root in findAllInMetadataBlockAndDataverse

---
 .../iq/dataverse/DatasetFieldServiceBean.java |  4 +++
 .../iq/dataverse/api/DataversesIT.java        | 29 +++++--------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
index c977ae784bd..e87d11dd7eb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
@@ -891,6 +891,10 @@ public List<DatasetFieldType> findAllDisplayedOnCreateInMetadataBlock(MetadataBl
     }
 
     public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate) {
+        if (!dataverse.isMetadataBlockRoot() && dataverse.getOwner() != null) {
+            return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate);
+        }
+
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
         CriteriaQuery<DatasetFieldType> criteriaQuery = criteriaBuilder.createQuery(DatasetFieldType.class);
 
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 9e3555555e8..f59d152f6be 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -911,27 +911,14 @@ public void testListMetadataBlocks() {
         createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
-        boolean issue10984fixed = false;
-        // See https://github.com/IQSS/dataverse/issues/10984
-        if (issue10984fixed) {
-            Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
-            listMetadataBlocks.prettyPrint();
-            listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
-            listMetadataBlocks.then().assertThat()
-                    .statusCode(OK.getStatusCode())
-                    .body("data[0].name", is("citation"))
-                    .body("data[0].fields.title.displayOnCreate", equalTo(true));
-
-        } else {
-            Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
-            listMetadataBlocks.prettyPrint();
-            listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
-            listMetadataBlocks.then().assertThat()
-                    .statusCode(OK.getStatusCode())
-                    .body("data[0].name", is("citation"))
-                    // "fields" should be more like 28, not 0
-                    .body("data[0].fields.size()", is(0));
-        }
+        Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
+        listMetadataBlocks.prettyPrint();
+        listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+        listMetadataBlocks.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].name", is("citation"))
+                .body("data[0].fields.title.displayOnCreate", equalTo(true))
+                .body("data[0].fields.size()", is(28));
 
         Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken);
         setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode());

From cfe9dbddb720a34e3360a18ace8808942bf93d22 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 31 Oct 2024 12:55:52 +0000
Subject: [PATCH 120/137] Removed: FIXME comment

---
 .../java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
index e87d11dd7eb..ded7c83de62 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
@@ -951,7 +951,7 @@ public List<DatasetFieldType> findAllInMetadataBlockAndDataverse(MetadataBlock m
         criteriaQuery.where(
                 criteriaBuilder.equal(dataverseRoot.get("id"), dataverse.getId()), // Match the Dataverse ID.
                 criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID.
-                metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse. FIXME: inherit blocks from parent
+                metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse.
                 datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock.
                 criteriaBuilder.or(includedAsInputLevelPredicate, hasNoInputLevelPredicate), // Include DatasetFieldTypes based on the input level predicates.
                 displayedOnCreatePredicate // Apply the display-on-create filter if necessary.

From f317ab005f241777fc4c8b4be531c7e17f9fdf54 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 31 Oct 2024 14:26:40 +0000
Subject: [PATCH 121/137] Added: tweaks to DataversesIT listMetadataBlocks

---
 src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index f59d152f6be..31a6c60bef9 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -911,11 +911,13 @@ public void testListMetadataBlocks() {
         createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
+        // New Dataverse should return just the citation block and its displayOnCreate fields when onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true
         Response listMetadataBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, apiToken);
         listMetadataBlocks.prettyPrint();
         listMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
         listMetadataBlocks.then().assertThat()
                 .statusCode(OK.getStatusCode())
+                .body("data.size()", equalTo(1))
                 .body("data[0].name", is("citation"))
                 .body("data[0].fields.title.displayOnCreate", equalTo(true))
                 .body("data[0].fields.size()", is(28));

From e7eace38ad31b2ab9995ac7f1485d477acd035c8 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 31 Oct 2024 10:52:12 -0400
Subject: [PATCH 122/137] Update FeatureFlags apiNotes

While adding a flag I noticed a typo and missing apiNote
---
 .../edu/harvard/iq/dataverse/settings/FeatureFlags.java     | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
index 33e828e619d..20632c170e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
@@ -97,12 +97,16 @@ public enum FeatureFlags {
      * for the dataset.
      * 
      * @apiNote Raise flag by setting
-     * "dataverse.feature.enable-dataset-thumbnail-autoselect"
+     * "dataverse.feature.disable-dataset-thumbnail-autoselect"
      * @since Dataverse 6.4
      */
     DISABLE_DATASET_THUMBNAIL_AUTOSELECT("disable-dataset-thumbnail-autoselect"),
     /**
      * Feature flag for the new Globus upload framework.
+     * 
+     * @apiNote Raise flag by setting
+     * "dataverse.feature.globus-use-experimental-async-framework"
+     * @since Dataverse 6.4
      */
     GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK("globus-use-experimental-async-framework"),
     ;

From 5b2b35d16cc4e76038343401af848c87809403d0 Mon Sep 17 00:00:00 2001
From: Omer Fahim <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 12:59:59 -0400
Subject: [PATCH 123/137] Update dev-usage.rst - Add Details for Accessing and
 Saving Harvesting Logs to Local

---
 .../source/container/dev-usage.rst            | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 6a1edcf7ebd..e481bcfd5ae 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -140,6 +140,48 @@ Alternatives:
 - If you used Docker Compose for running, you may use ``docker compose -f docker-compose-dev.yml logs <service name>``.
   Options are the same.
 
+Accessing Harvesting Log Files
+------------------------------
+
+1. Open a Terminal and Access Dataverse Container
+Run the following command to access the Dataverse container (assuming your container is named dataverse-1):
+
+.. code-block::
+docker exec -it dataverse-1 bash
+Code updated
+
+This command opens an interactive shell within the dataverse-1 container.
+
+2. Navigate to the Log Files Directory
+Once inside the container, navigate to the directory where Dataverse logs are stored:
+
+.. code-block::
+cd /opt/payara/appserver/glassfish/domains/domain1/logs
+Code updated
+
+This directory contains various log files, including those relevant to harvesting.
+
+3. Create a Directory for Copying Files
+Create a directory where you’ll copy the files you want to access on your local machine:
+
+mkdir /dv/filesToCopy
+
+This will create a new folder named filesToCopy inside /dv.
+
+4. Copy the Files to the New Directory
+Copy all files from the current directory to the newly created filesToCopy directory:
+
+cp * /dv/filesToCopy
+
+This command copies all files in the logs directory to /dv/filesToCopy.
+
+5. Access the Files on Your Mac
+On your Mac, the copied files should appear in the following directory:
+
+docker-dev-volumes/app/data/filesToCopy
+
+
+
 
 Redeploying
 -----------

From 90665f9ab705634d775656ab53f6b8c17823661e Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 1 Nov 2024 13:02:21 -0400
Subject: [PATCH 124/137] Revert "Bump actions/download-artifact from 3 to
 4.1.7 in /.github/workflows" (#10987)

will check after the reverted code is merged
---
 .github/workflows/deploy_beta_testing.yml | 2 +-
 .github/workflows/maven_unit_test.yml     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml
index efe3e0d8621..028f0140cc9 100644
--- a/.github/workflows/deploy_beta_testing.yml
+++ b/.github/workflows/deploy_beta_testing.yml
@@ -45,7 +45,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Download war artifact
-        uses: actions/download-artifact@v4.1.7
+        uses: actions/download-artifact@v3
         with:
           name: built-app
           path: ./
diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 102fb1d5882..a94b17a67ba 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -107,7 +107,7 @@ jobs:
                   cache: maven
 
             # Get the build output from the unit test job
-            - uses: actions/download-artifact@v4.1.7
+            - uses: actions/download-artifact@v3
               with:
                   name: java-artifacts
             - run: |
@@ -140,7 +140,7 @@ jobs:
                 cache: maven
 
           # Get the build output from the integration test job
-          - uses: actions/download-artifact@v4.1.7
+          - uses: actions/download-artifact@v3
             with:
                 name: java-reportdir
           - run: tar -xvf java-reportdir.tar

From 899cc25996e25a0ddd1cbb514ae912cf927eea9a Mon Sep 17 00:00:00 2001
From: Omer Fahim <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 14:17:10 -0400
Subject: [PATCH 125/137] Update
 doc/sphinx-guides/source/container/dev-usage.rst

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/container/dev-usage.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index e481bcfd5ae..7e4a640a45d 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -147,6 +147,7 @@ Accessing Harvesting Log Files
 Run the following command to access the Dataverse container (assuming your container is named dataverse-1):
 
 .. code-block::
+
 docker exec -it dataverse-1 bash
 Code updated
 

From 356da8fc09c429d8776698b4bc8aa81333cb56c4 Mon Sep 17 00:00:00 2001
From: Omer Fahim <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 14:21:04 -0400
Subject: [PATCH 126/137] Update
 doc/sphinx-guides/source/container/dev-usage.rst

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/container/dev-usage.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 7e4a640a45d..bd2ff6f0382 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -156,6 +156,7 @@ This command opens an interactive shell within the dataverse-1 container.
 2. Navigate to the Log Files Directory
 Once inside the container, navigate to the directory where Dataverse logs are stored:
 
+
 .. code-block::
 cd /opt/payara/appserver/glassfish/domains/domain1/logs
 Code updated

From 9119c4b4799fd4a14223ac9c07471ef8db34f095 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 14:30:37 -0400
Subject: [PATCH 127/137] Update dev-usage.rst

---
 doc/sphinx-guides/source/container/dev-usage.rst | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index bd2ff6f0382..7b5274844f6 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -149,7 +149,6 @@ Run the following command to access the Dataverse container (assuming your conta
 .. code-block::
 
 docker exec -it dataverse-1 bash
-Code updated
 
 This command opens an interactive shell within the dataverse-1 container.
 
@@ -159,28 +158,33 @@ Once inside the container, navigate to the directory where Dataverse logs are st
 
 .. code-block::
 cd /opt/payara/appserver/glassfish/domains/domain1/logs
-Code updated
 
 This directory contains various log files, including those relevant to harvesting.
 
 3. Create a Directory for Copying Files
 Create a directory where you’ll copy the files you want to access on your local machine:
 
-mkdir /dv/filesToCopy
+.. code-block::
+
+  mkdir /dv/filesToCopy
 
 This will create a new folder named filesToCopy inside /dv.
 
 4. Copy the Files to the New Directory
 Copy all files from the current directory to the newly created filesToCopy directory:
 
-cp * /dv/filesToCopy
+.. code-block::
+
+  cp * /dv/filesToCopy
 
 This command copies all files in the logs directory to /dv/filesToCopy.
 
 5. Access the Files on Your Mac
 On your Mac, the copied files should appear in the following directory:
 
-docker-dev-volumes/app/data/filesToCopy
+.. code-block::
+
+  docker-dev-volumes/app/data/filesToCopy
 
 
 

From 41a1d13288c31c938395bdab80601bc916391a15 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 14:39:21 -0400
Subject: [PATCH 128/137] Update dev-usage.rst

---
 doc/sphinx-guides/source/container/dev-usage.rst | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 7b5274844f6..da7d6845bb5 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -148,16 +148,16 @@ Run the following command to access the Dataverse container (assuming your conta
 
 .. code-block::
 
-docker exec -it dataverse-1 bash
+  docker exec -it dataverse-1 bash
 
 This command opens an interactive shell within the dataverse-1 container.
 
 2. Navigate to the Log Files Directory
 Once inside the container, navigate to the directory where Dataverse logs are stored:
 
-
 .. code-block::
-cd /opt/payara/appserver/glassfish/domains/domain1/logs
+
+  cd /opt/payara/appserver/glassfish/domains/domain1/logs
 
 This directory contains various log files, including those relevant to harvesting.
 
@@ -186,9 +186,6 @@ On your Mac, the copied files should appear in the following directory:
 
   docker-dev-volumes/app/data/filesToCopy
 
-
-
-
 Redeploying
 -----------
 

From 432feb0cba801a982b7d156b8d1d19a6625817a4 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 14:48:05 -0400
Subject: [PATCH 129/137] Update dev-usage.rst

---
 doc/sphinx-guides/source/container/dev-usage.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index da7d6845bb5..bfad3d34cf2 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -143,7 +143,7 @@ Alternatives:
 Accessing Harvesting Log Files
 ------------------------------
 
-1. Open a Terminal and Access Dataverse Container
+1. Open a terminal and access Dataverse container.  
 Run the following command to access the Dataverse container (assuming your container is named dataverse-1):
 
 .. code-block::
@@ -152,7 +152,7 @@ Run the following command to access the Dataverse container (assuming your conta
 
 This command opens an interactive shell within the dataverse-1 container.
 
-2. Navigate to the Log Files Directory
+2. Navigate to the Log Files directory
 Once inside the container, navigate to the directory where Dataverse logs are stored:
 
 .. code-block::

From 0d59b16f8c05ddc190f38efaf1891cf6750f8107 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 14:57:15 -0400
Subject: [PATCH 130/137] Update dev-usage.rst

---
 doc/sphinx-guides/source/container/dev-usage.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index bfad3d34cf2..27fbb28dd26 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -143,7 +143,7 @@ Alternatives:
 Accessing Harvesting Log Files
 ------------------------------
 
-1. Open a terminal and access Dataverse container.  
+1. Open a terminal and access the Dataverse container.  
 Run the following command to access the Dataverse container (assuming your container is named dataverse-1):
 
 .. code-block::
@@ -152,7 +152,7 @@ Run the following command to access the Dataverse container (assuming your conta
 
 This command opens an interactive shell within the dataverse-1 container.
 
-2. Navigate to the Log Files directory
+2. Navigate to the Log files directory.
 Once inside the container, navigate to the directory where Dataverse logs are stored:
 
 .. code-block::
@@ -161,7 +161,7 @@ Once inside the container, navigate to the directory where Dataverse logs are st
 
 This directory contains various log files, including those relevant to harvesting.
 
-3. Create a Directory for Copying Files
+3. Create a directory for copying files.
 Create a directory where you’ll copy the files you want to access on your local machine:
 
 .. code-block::
@@ -170,7 +170,7 @@ Create a directory where you’ll copy the files you want to access on your loca
 
 This will create a new folder named filesToCopy inside /dv.
 
-4. Copy the Files to the New Directory
+4. Copy the files to the new directory.
 Copy all files from the current directory to the newly created filesToCopy directory:
 
 .. code-block::
@@ -179,7 +179,7 @@ Copy all files from the current directory to the newly created filesToCopy direc
 
 This command copies all files in the logs directory to /dv/filesToCopy.
 
-5. Access the Files on Your Mac
+5. Access the files on Your Mac
 On your Mac, the copied files should appear in the following directory:
 
 .. code-block::

From c76d62a6200382e6b584e220acde64bfd33716d2 Mon Sep 17 00:00:00 2001
From: ofahimIQSS <mfahim11427@gmail.com>
Date: Fri, 1 Nov 2024 15:03:46 -0400
Subject: [PATCH 131/137] Update dev-usage.rst

---
 doc/sphinx-guides/source/container/dev-usage.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 27fbb28dd26..80e3cac989c 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -141,7 +141,7 @@ Alternatives:
   Options are the same.
 
 Accessing Harvesting Log Files
-------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 1. Open a terminal and access the Dataverse container.  
 Run the following command to access the Dataverse container (assuming your container is named dataverse-1):

From e28b6d342eddc97e2632a53cdca17cf789d58022 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 1 Nov 2024 15:27:17 -0400
Subject: [PATCH 132/137] tweaks #10996

---
 .../source/container/dev-usage.rst            | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 80e3cac989c..c02c1d4010f 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -143,7 +143,8 @@ Alternatives:
 Accessing Harvesting Log Files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-1. Open a terminal and access the Dataverse container.  
+\1. Open a terminal and access the Dataverse container.
+
 Run the following command to access the Dataverse container (assuming your container is named dataverse-1):
 
 .. code-block::
@@ -152,7 +153,8 @@ Run the following command to access the Dataverse container (assuming your conta
 
 This command opens an interactive shell within the dataverse-1 container.
 
-2. Navigate to the Log files directory.
+\2. Navigate to the log files directory.
+
 Once inside the container, navigate to the directory where Dataverse logs are stored:
 
 .. code-block::
@@ -161,8 +163,9 @@ Once inside the container, navigate to the directory where Dataverse logs are st
 
 This directory contains various log files, including those relevant to harvesting.
 
-3. Create a directory for copying files.
-Create a directory where you’ll copy the files you want to access on your local machine:
+\3. Create a directory for copying files.
+
+Create a directory where you'll copy the files you want to access on your local machine:
 
 .. code-block::
 
@@ -170,7 +173,8 @@ Create a directory where you’ll copy the files you want to access on your loca
 
 This will create a new folder named filesToCopy inside /dv.
 
-4. Copy the files to the new directory.
+\4. Copy the files to the new directory.
+
 Copy all files from the current directory to the newly created filesToCopy directory:
 
 .. code-block::
@@ -179,8 +183,9 @@ Copy all files from the current directory to the newly created filesToCopy direc
 
 This command copies all files in the logs directory to /dv/filesToCopy.
 
-5. Access the files on Your Mac
-On your Mac, the copied files should appear in the following directory:
+\5. Access the files on your local machine.
+
+On your local machine, the copied files should appear in the following directory:
 
 .. code-block::
 

From 052262fe5badf98395704773f6ddfc4a179d9897 Mon Sep 17 00:00:00 2001
From: jo-pol <jo-pol@users.noreply.github.com>
Date: Mon, 4 Nov 2024 10:25:02 +0100
Subject: [PATCH 133/137] replaced deprecated mime type with mp4

---
 .../propertyFiles/MimeTypeDetectionByFileExtension.properties   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
index 05e61a40c17..4507c22fdf8 100644
--- a/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
+++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
@@ -15,7 +15,7 @@ m=text/x-matlab
 mat=application/matlab-mat
 md=text/markdown
 mp3=audio/mp3
-m4a=audio/x-m4a
+m4a=audio/mp4
 nii=image/nii
 nc=application/netcdf
 ods=application/vnd.oasis.opendocument.spreadsheet

From a55d31f19f9d422e2160d15b68d9519c9e29d394 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 4 Nov 2024 16:19:17 +0000
Subject: [PATCH 134/137] Fixed: unit test assertion in JsonParserTest

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
index f241a5d1dda..236344a9200 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
@@ -283,7 +283,7 @@ public void parseDataverseDTO() throws JsonParseException {
             assertEquals("We do all the science.", actual.getDescription());
             assertEquals("LABORATORY", actual.getDataverseType().toString());
             assertEquals(2, actual.getDataverseContacts().size());
-            assertEquals("pi@example.edu,student@example.edu", actual.getDataverseContacts().get(0).getContactEmail());
+            assertEquals("pi@example.edu", actual.getDataverseContacts().get(0).getContactEmail());
             assertEquals(0, actual.getDataverseContacts().get(0).getDisplayOrder());
             assertEquals(1, actual.getDataverseContacts().get(1).getDisplayOrder());
         } catch (IOException ioe) {

From 1ed0d304307e8839493b00aa48cf1002ec8e5afa Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 4 Nov 2024 16:24:14 +0000
Subject: [PATCH 135/137] Added: assertion to JsonParserTest

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
index 236344a9200..52e9c6de678 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
@@ -284,6 +284,7 @@ public void parseDataverseDTO() throws JsonParseException {
             assertEquals("LABORATORY", actual.getDataverseType().toString());
             assertEquals(2, actual.getDataverseContacts().size());
             assertEquals("pi@example.edu", actual.getDataverseContacts().get(0).getContactEmail());
+            assertEquals("student@example.edu", actual.getDataverseContacts().get(1).getContactEmail());
             assertEquals(0, actual.getDataverseContacts().get(0).getDisplayOrder());
             assertEquals(1, actual.getDataverseContacts().get(1).getDisplayOrder());
         } catch (IOException ioe) {

From b1dcb00b8ad46549e7f74304b11e2dcc9d3a1e64 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 4 Nov 2024 16:25:50 +0000
Subject: [PATCH 136/137] Refactor: JsonParserTest.parseDataverseDTO

---
 .../iq/dataverse/util/json/JsonParserTest.java        | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
index 52e9c6de678..d1cb30e2bc3 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java
@@ -277,16 +277,17 @@ public void parseDataverseDTO() throws JsonParseException {
         try (FileReader reader = new FileReader("doc/sphinx-guides/source/_static/api/dataverse-complete.json")) {
             dvJson = Json.createReader(reader).readObject();
             DataverseDTO actual = sut.parseDataverseDTO(dvJson);
+            List<DataverseContact> actualDataverseContacts = actual.getDataverseContacts();
             assertEquals("Scientific Research", actual.getName());
             assertEquals("science", actual.getAlias());
             assertEquals("Scientific Research University", actual.getAffiliation());
             assertEquals("We do all the science.", actual.getDescription());
             assertEquals("LABORATORY", actual.getDataverseType().toString());
-            assertEquals(2, actual.getDataverseContacts().size());
-            assertEquals("pi@example.edu", actual.getDataverseContacts().get(0).getContactEmail());
-            assertEquals("student@example.edu", actual.getDataverseContacts().get(1).getContactEmail());
-            assertEquals(0, actual.getDataverseContacts().get(0).getDisplayOrder());
-            assertEquals(1, actual.getDataverseContacts().get(1).getDisplayOrder());
+            assertEquals(2, actualDataverseContacts.size());
+            assertEquals("pi@example.edu", actualDataverseContacts.get(0).getContactEmail());
+            assertEquals("student@example.edu", actualDataverseContacts.get(1).getContactEmail());
+            assertEquals(0, actualDataverseContacts.get(0).getDisplayOrder());
+            assertEquals(1, actualDataverseContacts.get(1).getDisplayOrder());
         } catch (IOException ioe) {
             throw new JsonParseException("Couldn't read test file", ioe);
         }

From 368bcc9ddced52d0b6b07f2a571fbd10ca1177ec Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Nov 2024 10:48:45 -0500
Subject: [PATCH 137/137] add "garbled affiliation" test case #8573

---
 .../source/developers/remote-users.rst          | 17 +++++++++++------
 .../source/installation/shibboleth.rst          |  2 ++
 .../providers/shib/ShibServiceBean.java         |  5 +++++
 .../authorization/providers/shib/ShibUtil.java  | 13 +++++++++++++
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst
index 38b3edab772..35551b4e75b 100755
--- a/doc/sphinx-guides/source/developers/remote-users.rst
+++ b/doc/sphinx-guides/source/developers/remote-users.rst
@@ -2,22 +2,27 @@
 Shibboleth, OAuth and OIDC
 ==========================
 
+If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth, OAuth, and OIDC users. For some background on user accounts in the Dataverse Software, see :ref:`auth-modes` section of Configuration in the Installation Guide.
+
 .. contents:: |toctitle|
 	:local:
 
-Shibboleth and OAuth
---------------------
-
-If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth and OAuth users. For some background on user accounts in the Dataverse Software, see :ref:`auth-modes` section of Configuration in the Installation Guide.
+Shibboleth
+----------
 
-Rather than setting up Shibboleth on your laptop, developers are advised to add the Shibboleth auth provider (see "Add the Shibboleth Authentication Provider to Your Dataverse Installation" at :doc:`/installation/shibboleth`) and add a value to their database to enable Shibboleth "dev mode" like this:
+Rather than setting up Shibboleth on your laptop, developers are advised to add the Shibboleth auth provider (see :ref:`add-shib-auth`) and add a value to their database to enable Shibboleth "dev mode" like this:
 
 ``curl http://localhost:8080/api/admin/settings/:DebugShibAccountType -X PUT -d RANDOM``
 
-For a list of possible values, please "find usages" on the settings key above and look at the enum.
+For a list of possible values, please "find usages" on the settings key above and look at the enum. As of this writing it's at the top of ShibServiceBean.
 
 Now when you go to http://localhost:8080/shib.xhtml you should be prompted to create a Shibboleth account.
 
+If you want to test again with the same account, you can delete it as long as you didn't create anything with it. See :ref:`delete-a-user`.
+
+OAuth
+-----
+
 OAuth is much more straightforward to get working on your laptop than Shibboleth. GitHub is a good identity provider to test with because you can easily request a Client ID and Client Secret that works against localhost. Follow the instructions in the :doc:`/installation/oauth2` section of the installation Guide and use "http://localhost:8080/oauth2/callback.xhtml" as the callback URL.
 
 In addition to setting up OAuth on your laptop for real per above, you can also use a dev/debug mode:
diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst
index d5086d07a0e..5a1700f1a99 100644
--- a/doc/sphinx-guides/source/installation/shibboleth.rst
+++ b/doc/sphinx-guides/source/installation/shibboleth.rst
@@ -285,6 +285,8 @@ As a sanity check, visit the following URLs (substituting your hostname) to make
 
 The JSON in ``DiscoFeed`` comes from the list of IdPs you configured in the ``MetadataProvider`` section of ``shibboleth2.xml`` and will form a dropdown list on the Login Page.
 
+.. _add-shib-auth:
+
 Add the Shibboleth Authentication Provider to Your Dataverse Installation
 -------------------------------------------------------------------------
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibServiceBean.java
index 0921b2c6683..0814235db84 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibServiceBean.java
@@ -60,6 +60,7 @@ public enum DevShibAccountType {
         MISSING_REQUIRED_ATTR,
         ONE_AFFILIATION,
         TWO_AFFILIATIONS,
+        GARBLED_AFFILIATION,
     };
 
     public DevShibAccountType getDevShibAccountType() {
@@ -150,6 +151,10 @@ public void possiblyMutateRequestInDev(HttpServletRequest request) {
                 ShibUtil.mutateRequestForDevConstantTwoAffiliations(request);
                 break;
 
+            case GARBLED_AFFILIATION:
+                ShibUtil.mutateRequestForDevConstantGarbledAffiliation(request);
+                break;
+
             default:
                 logger.info("Should never reach here");
                 break;
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
index 4cf41903405..00cac4b9661 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
@@ -13,6 +13,8 @@
 import java.util.UUID;
 import java.util.logging.Logger;
 import jakarta.servlet.http.HttpServletRequest;
+import java.nio.charset.StandardCharsets;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
 
 public class ShibUtil {
 
@@ -300,6 +302,17 @@ static void mutateRequestForDevConstantTwoAffiliations(HttpServletRequest reques
         request.setAttribute("ou", "SNPP;Stonecutters");
     }
 
+    static void mutateRequestForDevConstantGarbledAffiliation(HttpServletRequest request) {
+        request.setAttribute(ShibUtil.shibIdpAttribute, "https://fake.example.com/idp/shibboleth");
+        request.setAttribute(ShibUtil.uniquePersistentIdentifier, "uzorker");
+        request.setAttribute(ShibUtil.firstNameAttribute, "Üter");
+        request.setAttribute(ShibUtil.lastNameAttribute, "Zörker");
+        request.setAttribute(ShibUtil.emailAttribute, "uzorker@mailinator.com");
+        request.setAttribute(ShibUtil.usernameAttribute, "uzorker");
+        // Affiliation. "ou" is the suggested attribute in :ShibAffiliationAttribute.
+        request.setAttribute("ou", new String("Universität".getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1));
+    }
+
     public static Map<String, String> getRandomUserStatic() {
         Map<String, String> fakeUser = new HashMap<>();
         String shortRandomString = UUID.randomUUID().toString().substring(0, 8);