Merge pull request #8 from sem42198/working_branch

Working branch

Author: lmcglohon

README.md

@@ -16,3 +16,12 @@ We recommend migrating to the latest version of ArchivesSpace when possible. Arc
 5. Change the default information in the migration tool user interface as needed.
 6. Press the "Copy To Archives Space" button to start the migration process.
 7. Assess the migration using the migration report and clean up data as needed.
+
+####Notes For Developers
+
+To build this tool, use the IDE of your choice to build a standalone jar executable. Most of development work will probably take place on the following classes.
+
+* ASpaceCopyUtil - This class essentially controls the migration process and interacts with the UI (dbCopyFrame) as well as the classes listed below.
+* ASpaceMapper - Converts Archon JSON records to the ArchivesSpace JSON model.
+* ASpaceClient - Saves records to ASpace through http post.
+* ASpaceEnumUtil - Converts an Archon enum ID to the corresponding ASpace enum. Usually called by ASpaceMapper.

docs/Archon to ASpace Data Mapping.xlsx

@@ -610,7 +610,7 @@ private void initComponents() {
                     }));
 
                 //---- apiLabel ----
-                apiLabel.setText("  Archives Space Version: v1.4.x");
+                apiLabel.setText("  Archives Space Version: v2.x");
                 apiLabel.setHorizontalTextPosition(SwingConstants.CENTER);
                 contentPanel.add(apiLabel, cc.xy(1, 1));
 

docs/Archon_Migration_Guidelines.docx

@@ -199,6 +199,11 @@ public String getASpaceNameSource(int arID) {
         return getEnumValueForID(key, "local");
     }
 
+    public String getASpaceNameSourceForSubject(int arID) {
+        String key = "name_subject_source_" + arID;
+        return getEnumValueForID(key, "local");
+    }
+
     /**
      * Method to initASpaceialize the name rules array
      */
@@ -1235,8 +1240,9 @@ public void setASpaceDynamicEnums(HashMap<String, JSONObject> dynamicEnums) {
      * @param listName
      * @return
      */
-    public JSONObject getDynamicEnum(String listName) throws Exception {
+    public ArrayList<JSONObject> getDynamicEnum(String listName) throws Exception {
         JSONObject dynamicEnum = null;
+        JSONObject dynamicEnum2 = null;
 
         if (listName.contains("countries")) {
             dynamicEnum = dynamicEnums.get("country_iso_3166");
@@ -1275,9 +1281,17 @@ public JSONObject getDynamicEnum(String listName) throws Exception {
             dynamicEnum = dynamicEnums.get("subject_source");
             dynamicEnum.put("valueKey", "EADSource");
             dynamicEnum.put("idPrefix", "subject_source");
+            // since some subjects get mapped to agents we need the Archon's subject sources for agents as well as subjects
+            dynamicEnum2 = dynamicEnums.get("name_source");
+            dynamicEnum2.put("valueKey", "EADSource");
+            dynamicEnum2.put("idPrefix", "name_subject_source");
         }
 
-        return dynamicEnum;
+        ArrayList<JSONObject> dynamicEnums = new ArrayList<JSONObject>();
+        dynamicEnums.add(dynamicEnum);
+        dynamicEnums.add(dynamicEnum2);
+
+        return dynamicEnums;
     }
 
     /**

src/org/nyu/edu/dlts/dbCopyFrame.java

@@ -54,9 +54,6 @@ public class ASpaceMapper {
 
     // date formatter used to convert date string to date object
     SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
-    SimpleDateFormat simpleDateFormat2 = new SimpleDateFormat("yyyyddMM");
-    SimpleDateFormat humanDateFormat = new SimpleDateFormat("MM/dd/yyyy");
-
 
     // booleans used to convert some bbcode to html or blanks
     private boolean bbcodeToHTML = false;
@@ -128,68 +125,72 @@ public ASpaceEnumUtil getEnumUtil() {
      * @param enumList
      * @return
      */
-    public JSONObject mapEnumList(JSONObject enumList, String endpoint) throws Exception {
+    public ArrayList<JSONObject> mapEnumList(JSONObject enumList, String endpoint) throws Exception {
         // first we get the correct dynamic enum based on list. If it null then we just return null
-        JSONObject dynamicEnumJS = enumUtil.getDynamicEnum(endpoint);
-
-        if(dynamicEnumJS == null) return null;
+        ArrayList<JSONObject> dynamicEnums = enumUtil.getDynamicEnum(endpoint);
+        ArrayList<JSONObject> dynamicEnumsUpdated = new ArrayList<JSONObject>();
 
-        // add the values return from aspace enum an arraylist to make lookup easier
-        JSONArray valuesJA = dynamicEnumJS.getJSONArray("values");
-        ArrayList<String> valuesList = new ArrayList<String>();
-        for(int i = 0; i < valuesJA.length(); i++) {
-            valuesList.add(valuesJA.getString(i));
-        }
+        for (JSONObject dynamicEnumJS : dynamicEnums) {
 
-        // now see if all the archon values are in the ASpace list already
-        // if they are not then add them
-        String valueKey = dynamicEnumJS.getString("valueKey");
-        String idPrefix = dynamicEnumJS.getString("idPrefix");
+            if (dynamicEnumJS == null) return dynamicEnumsUpdated;
 
-        boolean toLowerCase = true;
-        if(dynamicEnumJS.has("keepValueCase")) {
-            toLowerCase = false;
-        }
+            // add the values return from aspace enum an arraylist to make lookup easier
+            JSONArray valuesJA = dynamicEnumJS.getJSONArray("values");
+            ArrayList<String> valuesList = new ArrayList<String>();
+            for (int i = 0; i < valuesJA.length(); i++) {
+                valuesList.add(valuesJA.getString(i));
+            }
 
-        int count = 0;
-        Iterator<String> keys = enumList.keys();
-        while(keys.hasNext()) {
-            JSONObject enumJS = enumList.getJSONObject(keys.next());
-            String value = enumJS.getString(valueKey);
+            // now see if all the archon values are in the ASpace list already
+            // if they are not then add them
+            String valueKey = dynamicEnumJS.getString("valueKey");
+            String idPrefix = dynamicEnumJS.getString("idPrefix");
 
-            // most values in ASpace are lower case so normalize
-            if(toLowerCase) {
-                value = value.toLowerCase();
+            boolean toLowerCase = true;
+            if (dynamicEnumJS.has("keepValueCase")) {
+                toLowerCase = false;
             }
 
-            // some values have spaces which space normally uses underscore for
-            value = value.replace(" ", "_");
+            int count = 0;
+            Iterator<String> keys = enumList.keys();
+            while (keys.hasNext()) {
+                JSONObject enumJS = enumList.getJSONObject(keys.next());
+                String value = enumJS.getString(valueKey);
+
+                // most values in ASpace are lower case so normalize
+                if (toLowerCase) {
+                    value = value.toLowerCase();
+                }
+
+                // some values have spaces which space normally uses underscore for
+                value = value.replace(" ", "_");
 
-            // map the id to value
-            String id = idPrefix + "_" + enumJS.get("ID");
-            enumUtil.addIdAndValueToEnumList(id, value);
-            if (idPrefix.equals("container_types")) {
-                aspaceCopyUtil.addContainerTypeValueToIDMapping(value, enumJS.getString("ID"));
+                // map the id to value
+                String id = idPrefix + "_" + enumJS.get("ID");
+                enumUtil.addIdAndValueToEnumList(id, value);
+                if (idPrefix.equals("container_types")) {
+                    aspaceCopyUtil.addContainerTypeValueToIDMapping(value, enumJS.getString("ID"));
+                }
+
+                // see if to add this to aspace
+                if (!valuesList.contains(value)) {
+                    valuesJA.put(value);
+                    count++;
+                    System.out.println("Adding value " + value);
+                }
             }
 
-            // see if to add this to aspace
-            if(!valuesList.contains(value)) {
-                valuesJA.put(value);
-                count++;
-                System.out.println("Adding value " + value);
+            // need to add other to extent unit type enum list
+            if (endpoint.contains("extentunits")) {
+                valuesJA.put(ASpaceEnumUtil.UNMAPPED);
             }
-        }
 
-        // need to add other to extent unit type enum list
-        if(endpoint.contains("extentunits")) {
-            valuesJA.put(ASpaceEnumUtil.UNMAPPED);
-        }
 
-        if(count != 0) {
-            return dynamicEnumJS;
-        } else {
-            return null;
+            if (count != 0) {
+                dynamicEnumsUpdated.add(dynamicEnumJS);
+            }
         }
+        return dynamicEnumsUpdated;
     }
 
     /**
@@ -215,6 +216,7 @@ public String getCorporateAgent(JSONObject repository) throws JSONException {
         contactsJS.put("address_1", repository.get("Address"));
         contactsJS.put("address_2", repository.get("Address2"));
         contactsJS.put("city", repository.get("City"));
+        contactsJS.put("region", repository.get("State"));
 
         // add the country and country code together
         contactsJS.put("country", enumUtil.getASpaceCountryCode(repository.getInt("CountryID")));
@@ -292,6 +294,7 @@ public String convertRepository(JSONObject record, String agentURI) throws Excep
         json.put("org_code", record.get("Code"));
         json.put("url", fixUrl(record.getString("URL")));
         json.put("publish", true);
+        json.put("country", enumUtil.getASpaceCountryCode(record.getInt("CountryID")));
 
         if(agentURI != null) {
             json.put("agent_representation", getReferenceObject(agentURI));
@@ -422,7 +425,9 @@ public JSONObject convertCreator(JSONObject record, int creatorTypeId) throws Ex
         }
 
         // add the source for the name
-        namesJS.put("source", enumUtil.getASpaceNameSource(record.getInt("CreatorSourceID")));
+        if (record.has("SubjectSourceID")) {
+            namesJS.put("source", enumUtil.getASpaceNameSourceForSubject(record.getInt("SubjectSourceID")));
+        } else namesJS.put("source", enumUtil.getASpaceNameSource(record.getInt("CreatorSourceID")));
 
         // add basic information to the names record
         String sortName = record.getString("Name");
@@ -897,11 +902,12 @@ public HashMap<String, String> getFileIDsToFilenamesMap() {
      * Method to convert an collection record to json ASpace JSON
      *
      * @param record
-     * @param classificationIdPartsMap
+     * @param classificationIdentifiers
      * @return
      * @throws Exception
      */
-    public JSONObject convertCollection(JSONObject record, HashMap<String, String> classificationIdPartsMap) throws Exception {
+    public JSONObject convertCollection(JSONObject record, HashMap<String, String> classificationIdentifiers,
+                                        HashMap<String, String> classificationParents) throws Exception {
         // Main json object
         JSONObject json = new JSONObject();
 
@@ -934,30 +940,35 @@ public JSONObject convertCollection(JSONObject record, HashMap<String, String> c
 
         // get the ids and make them unique if we in DEBUG mode
         String id = record.getString("CollectionIdentifier");
+
+        // if the collection ID is empty fix it
+        if (id == null || id.isEmpty()) {
+            id = "##" + randomString.nextString();
+            while (resourceIDs.contains(id)) {
+                id = "##" + randomString.nextString();
+            }
+            aspaceCopyUtil.addErrorMessage("Empty collection ID. Changed to " + id + "\n");
+        }
+
         String classificationID = record.getString("ClassificationID");
         String[] idParts = new String[]{"", "", "", ""};
 
-        if(!classificationID.equals("0") && classificationIdPartsMap.get(classificationID) != null) {
-            String[] sa = classificationIdPartsMap.get(classificationID).split("/");
-
-            // this can be placed in a loop but lets keep it nice an clear?
-            if(sa.length == 1) {
-                idParts[0] = sa[0];
-                idParts[1]  = id;
-            } else if(sa.length == 2) {
-                idParts[0] = sa[0];
-                idParts[1] = sa[1];
-                idParts[2] = id;
-            } else if (sa.length == 3) {
-                idParts[0] = sa[0];
-                idParts[1] = sa[1];
-                idParts[2] = sa[2];
-                idParts[3] = id;
-            }
-        } else {
-            idParts[0] = id;
+        Stack<String> fullId = new Stack<String>();
+        fullId.push(id);
+        String cId = classificationID;
+        while (cId != null) {
+            String identifier = classificationIdentifiers.get(cId);
+            if (identifier == null) break;
+            fullId.push(identifier);
+            cId = classificationParents.get(cId);
         }
 
+        idParts[0] = fullId.pop();
+        if (!fullId.isEmpty()) idParts[1] = fullId.pop();
+        if (!fullId.isEmpty()) idParts[2] = fullId.pop();
+        while (fullId.size() > 1) idParts[2] += "-" + fullId.pop();
+        if (!fullId.isEmpty()) idParts[3] = fullId.pop();
+
         // make sure the id is unique
         getUniqueID(ASpaceClient.RESOURCE_ENDPOINT, "", idParts, title);
 
@@ -1215,58 +1226,12 @@ public JSONObject convertCollectionContent(JSONObject record) throws Exception {
             json.put("component_id", record.getString("UniqueID"));
         }
 
-        //json.put("position", record.getInt("SortOrder"));
-
-        // place a dummy key to help with the sorting if we redoing the sorting
-        String paddedSortOrder = String.format("%05d", record.getInt("SortOrder"));
-        json.put("sort_key1", paddedSortOrder + "_" + record.get("ID"));
-        json.put("sort_key2", ""); // sort order of the parent physical only content record
-
         // add the notes
         addResourceComponentNotes(record, json);
 
         return json;
     }
 
-    /**
-     * Method to convert container information for physical only component
-     * into a component
-     *
-     * @param aoEndpoint
-     * @param containerList
-     * @return
-     * @throws Exception
-     */
-    public JSONObject convertContainerInformation(String aoEndpoint, ArrayList<String> containerList) throws Exception {
-        String[] sa = containerList.get(0).split("::");
-
-        // Main json object
-        JSONObject json = new JSONObject();
-
-        json.put("uri", aoEndpoint + "/" + sa[2]);
-        json.put("jsonmodel_type", "archival_object");
-
-        json.put("publish", publishRecord);
-
-
-        /* Add fields needed for abstract_archival_object.rb */
-
-        // check to make sure we have a title
-        String title = WordUtils.capitalize(sa[0] + " " + sa[1]);
-        json.put("title", title);
-
-        /* add field required for archival_object.rb */
-        json.put("level", "item");
-
-        // make the ref id unique otherwise ASpace complains
-        String refId = sa[2];
-        json.put("ref_id", refId);
-
-        json.put("position", new Integer(sa[3]));
-
-        return json;
-    }
-
     /**
      * Method to add a date json object
      *
@@ -1480,7 +1445,17 @@ private void addSinglePartNote(JSONArray notesJA, String noteType, String noteLa
      * @throws Exception
      */
     private void addMultipartNote(JSONArray notesJA, String noteType, String noteLabel, String noteContent) throws Exception {
-        if(noteContent.isEmpty() || noteType.isEmpty()) return;
+        if(noteContent.trim().isEmpty() || noteType.isEmpty()) return;
+
+        // these note types don't exist in ASpace
+        if (noteType.equals("unitid") || noteType.equals("origination") || noteType.equals("note")) noteType = "odd";
+
+        // these note types should be single part
+        if (noteType.equals("physfacet") || noteType.equals("physdesc") || noteType.equals("langmaterial") ||
+                noteType.equals("materialspec")) {
+            addSinglePartNote(notesJA, noteType, noteLabel, noteContent);
+            return;
+        }
 
         JSONObject noteJS = new JSONObject();
 
@@ -1857,19 +1832,12 @@ private Date getDate(String dateString) {
      * @return
      */
     private String getHumanReadableDate(String dateString) {
-        Date date = null;
-
         try {
-            date = simpleDateFormat.parse(dateString);
-        } catch (ParseException e) { }
-
-        try {
-            date = simpleDateFormat2.parse(dateString);
-        } catch (ParseException e) { }
-
-        if(date != null) {
-            return humanDateFormat.format(date);
-        } else {
+            String year = dateString.substring(0, 4);
+            String month = dateString.substring(4, 6);
+            String day = dateString.substring(6);
+            return month + "/" + day + "/" + year;
+        } catch (Exception e) {
             return dateString;
         }
     }