Merge pull request #63 from salesforce/jdk-17

fixed locale changes by JDK 17

Author: yoikawa

src/main/java/com/force/i18n/LabelUtils.java

@@ -130,7 +130,7 @@ public static String getSampleGrammarFile(@Nullable String grammar) {
         } else {
             // If they pasted in a whole file, it's fine.
             if (!grammar.contains("<sfdcnames>") && !grammar.contains("<sfdcadjectives>")
-            		&& !grammar.contains("<names>") && !grammar.contains("<adjectives>")) {
+                    && !grammar.contains("<names>") && !grammar.contains("<adjectives>")) {
                 StringBuilder sb = new StringBuilder(grammar.length() + 100);
                 sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
                 sb.append("<names>");
@@ -139,7 +139,7 @@ public static String getSampleGrammarFile(@Nullable String grammar) {
                 return sb.toString();
             } else {
                 // Remove the dtd which confuses the parser
-            	String fixedGrammar = grammar.replace("<!DOCTYPE sfdcnames SYSTEM \"sfdcnames.dtd\">", "");
+                String fixedGrammar = grammar.replace("<!DOCTYPE sfdcnames SYSTEM \"sfdcnames.dtd\">", "");
                 fixedGrammar = fixedGrammar.replace("<!DOCTYPE sfdcadjectives SYSTEM \"sfdcadjectives.dtd\">", "");
                 fixedGrammar = fixedGrammar.replace("<!DOCTYPE names SYSTEM \"names.dtd\">", "");
                 fixedGrammar = fixedGrammar.replace("<!DOCTYPE adjectives SYSTEM \"adjectives.dtd\">", "");
@@ -180,46 +180,63 @@ public String nounify(String input, LanguageDictionary dictionary) {
         return new Nounifier(dictionary).nounifyString(input);
     }
 
-    /**
-     * Set of locale langauges strings where the results are JDK dependent
+    /*
+     * Map of locale langauges strings where the results are JDK dependent.
+     * Java 17 fixed legacy language code and autmatically replace it in {@code java.util.Locale} (JDK-8267069). This is
+     * for backward compatibility.
      */
-    static final Set<String> JDK_DEPENDENT_LANGUAGE = ImmutableSet.of(LanguageConstants.YIDDISH_ISO,
-    		LanguageConstants.HEBREW_ISO, LanguageConstants.INDONESIAN_ISO);
-
+    static final Map<String, String> JDK_DEPENDENT_LANGUAGE = Map.of( //
+           LanguageConstants.YIDDISH_ISO, LanguageConstants.YIDDISH, //
+           LanguageConstants.HEBREW_ISO, LanguageConstants.HEBREW, //
+           LanguageConstants.INDONESIAN_ISO, LanguageConstants.INDONESIAN);
 
-    public static List<URL> getFileNames(HumanLanguage language,URL rootDirectory, String basename ) {
+    public static List<URL> getFileNames(HumanLanguage language, URL rootDirectory, String basename ) {
         List<URL> list = new ArrayList<>();
         try {
-    	    Locale locale = language.getLocale();
-            list.add(new URL(rootDirectory,  locale.getLanguage() + '/' + basename));
+            final Locale locale = language.getLocale();
+            final String override = language.getOverrideLanguage();
+            final boolean hasOverride = override != null && !locale.getLanguage().equals(override);
+
+            list.add(getUrl(rootDirectory, locale.getLanguage(), basename));
+            // If the override language is set, add it
+            if (hasOverride) list.add(getUrl(rootDirectory, override, basename));
             // If the iso code for the language changed between JDK versions, use the old isocode for the directory
-            if (JDK_DEPENDENT_LANGUAGE.contains(locale.getLanguage())) {
-                list.add(new URL(rootDirectory,  language.getOverrideLanguage() + '/' + basename));
+            if (JDK_DEPENDENT_LANGUAGE.containsKey(locale.getLanguage())) {
+                list.add(getUrl(rootDirectory, JDK_DEPENDENT_LANGUAGE.get(locale.getLanguage()), basename));
+            }
+
+            if (!locale.getCountry().isEmpty()) {
+                /*
+                 * Code required because there is no Chinese traditional locale in jdk Locale.java and taiwan and Hong
+                 * kong are 2 different countries
+                 * We will assume that Hong Kong (HK country code) derive the Traditional Chinese from Taiwan (TW)
+                 * In the future, to define Hong Kong label, a new directory will need to be created zh/HK.
+                 */
+                if (language.getLocaleString().equals(LanguageConstants.CHINESE_HK)) {
+                    // adding Taiwan as fallback. assuming no override, or variant are set for this
+                    list.add(getUrl(rootDirectory, locale.getLanguage(), Locale.TRADITIONAL_CHINESE.getCountry(), basename));
+                    list.add(getUrl(rootDirectory, locale.getLanguage(), locale.getCountry(), basename));
+                } else {
+                    list.add(getUrl(rootDirectory, locale.getLanguage(), locale.getCountry(), basename));
+                    if (hasOverride) list.add(getUrl(rootDirectory, override, locale.getCountry(), basename));
+
+                    if (!locale.getVariant().isEmpty()) {
+                        list.add(getUrl(rootDirectory, locale.getLanguage(), locale.getCountry(), locale.getVariant(), basename));
+                        if (hasOverride) list.add(getUrl(rootDirectory, override, locale.getCountry(), locale.getVariant(), basename));
+                    }
+                }
             }
-    	    if (locale.getCountry().length() > 0) {
-    	        /*
-    	         * Code required because there is no Chinese traditional locale in jdk Locale.java and taiwan and Hong kong are 2 different countries
-    	         * We will assume that Hong Kong (HK country code) derive the Traditional Chinese from Taiwan (TW)
-    	         * In the future, to define Hong Kong label, a new directory will need to be created zh/HK.
-    	         */
-    	        if(language.getLocaleString().equals(LanguageConstants.CHINESE_HK)){
-    	            //adding Taiwan as fallback
-    	            list.add(new URL(rootDirectory, locale.getLanguage() + '/'+ Locale.TRADITIONAL_CHINESE.getCountry() +'/'+ basename));
-    	            list.add(new URL(rootDirectory, locale.getLanguage() + '/'+ locale.getCountry() + '/'+ basename));
-    	        }else{
-    	            list.add(new URL(rootDirectory, locale.getLanguage() + '/' + locale.getCountry() + '/' + basename));
-    	            if (locale.getVariant().length() > 0) {
-    	                list.add(new URL(rootDirectory, locale.getLanguage() + '/' + locale.getCountry() + '/' + locale.getVariant() + '/' + basename));
-    	            }
-    	        }
-    	    }
         } catch (MalformedURLException e) {
             throw new IllegalArgumentException(e);
         }
-	    return list;
-	}
+        return list;
+    }
+
+    private static URL getUrl(URL root, String... paths) throws MalformedURLException {
+        return new URL(root, String.join("/", paths));
+    }
 
-	private static final Pattern nonalphaPattern = Pattern.compile("\\W");
+    private static final Pattern nonalphaPattern = Pattern.compile("\\W");
 
     // TODO: Switch this to splitter.
     static List<String> tokenize(String input) {
@@ -244,7 +261,7 @@ static List<String> tokenize(String input) {
      * to determine if there should be some "tokenization" of the string.
      */
     public static class Nounifier {
-        private final static Set<String> EXCLUDED_NOUNS = ImmutableSet.of("Role", "Email", "Address", "{0}");
+        private static final Set<String> EXCLUDED_NOUNS = ImmutableSet.of("Role", "Email", "Address", "{0}");
 
         private final GenericTrieMatcher<String> nounMatcher;
         private final GenericTrieMatcher<String> adjMatcher;

src/main/java/com/force/i18n/commons/util/collection/IntHashMap.java

@@ -8,7 +8,6 @@
 package com.force.i18n.commons.util.collection;
 
 import java.io.IOException;
-import java.io.Serializable;
 import java.util.*;
 
 import com.google.common.annotations.Beta;
@@ -77,12 +76,12 @@
  * Beta class. Generally use {@link java.util.HashMap} or {@link java.util.EnumMap} instead.
  *
  * @author  Based on Sun's java.util.HashMap (modified by koliver)
- * @see	    IntMap
- * @see	    java.util.HashMap
+ * @see        IntMap
+ * @see        java.util.HashMap
  */
 @Beta
 @SuppressWarnings("rawtypes") // TODO Fix
-public class IntHashMap<V> extends AbstractIntMap<V> implements Serializable {
+public class IntHashMap<V> extends AbstractIntMap<V> {
     private static final long serialVersionUID = 0L;
 
     private static final int DEFAULT_CAPACITY = 101;
@@ -186,7 +185,7 @@ public int size() {
      *
      * @param value value whose presence in this map is to be tested.
      */
-	@Override
+    @Override
     public boolean containsValue(Object value) {
         IEntry[] tab = this.table;
 
@@ -287,9 +286,9 @@ private void rehash() {
      * @param key key with which the specified value is to be associated.
      * @param value value to be associated with the specified key.
      * @return previous value associated with specified key, or {@code null}
-     *	       if there was no mapping for key.  A {@code null} return can
-     *	       also indicate that the HashMap previously associated
-     *	       {@code null} with the specified key.
+     *           if there was no mapping for key.  A {@code null} return can
+     *           also indicate that the HashMap previously associated
+     *           {@code null} with the specified key.
      */
     @Override
     public V put(int key, V value) {
@@ -328,9 +327,9 @@ public V put(int key, V value) {
      *
      * @param key key whose mapping is to be removed from the map.
      * @return previous value associated with specified key, or {@code null}
-     *	       if there was no mapping for key.  A {@code null} return can
-     *	       also indicate that the map previously associated {@code null}
-     *	       with the specified key.
+     *           if there was no mapping for key.  A {@code null} return can
+     *           also indicate that the map previously associated {@code null}
+     *           with the specified key.
      */
     @Override
     public V remove(int key) {
@@ -734,6 +733,20 @@ public int next() {
         }
     }
 
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof IntHashMap) {
+            // super compares table and count. that may be enough
+            return super.equals(other);
+        }
+        return false;
+    }
+
     // Serialization
     private void writeObject(java.io.ObjectOutputStream s) throws IOException {
         // Write out the threshold, loadfactor, and any hidden stuff

src/main/java/com/force/i18n/grammar/LanguageDictionary.java

@@ -64,7 +64,7 @@ public class LanguageDictionary implements Serializable {
     // TODO: These could all be made lists when serialized
 
     // map to Noun
-    protected GrammaticalTermMap<Noun> nounMap; 
+    protected GrammaticalTermMap<Noun> nounMap;
     // map to Noun
     protected GrammaticalTermMap<Noun>  nounMapByPluralAlias;
     // map to Adjective
@@ -202,9 +202,9 @@ public GrammaticalTermMap<Noun> getNounByPluralAlias() {
     public GrammaticalTermMap<Adjective> getAdjectiveMap() {
         return adjectiveMap;
     }
-    
+
     /**
-     * get termMap of Article 
+     * get termMap of Article
      */
     public GrammaticalTermMap<Article> getArticleMap() {
         return articleMap;
@@ -213,10 +213,10 @@ public GrammaticalTermMap<Article> getArticleMap() {
 
     private void forAllTerms (TermType type, BiConsumer<String, GrammaticalTerm> f) {
         switch (type) {
-        case Noun: 
+        case Noun:
             for(Map.Entry<String, Noun> e : nounMap.entrySet()) f.accept(e.getKey(), e.getValue());
             break;
-        case Adjective: 
+        case Adjective:
             for(Map.Entry<String, Adjective> e : adjectiveMap.entrySet()) f.accept(e.getKey(), e.getValue());
             break;
         case Article:
@@ -226,14 +226,14 @@ private void forAllTerms (TermType type, BiConsumer<String, GrammaticalTerm> f)
             throw new AssertionError("Invalid term type " + type);
         }
     }
-    
+
 
 
     public Set<String> getAllTermNames(TermType type) {
         switch (type) {
-        case Noun: 
+        case Noun:
             return Collections.unmodifiableSet(nounMap.keySet());
-        case Adjective: 
+        case Adjective:
             return Collections.unmodifiableSet(adjectiveMap.keySet());
         case Article:
             return Collections.unmodifiableSet(articleMap.keySet());
@@ -249,8 +249,8 @@ public Set<String> getAllTermNames(TermType type) {
     public Set<String> getAllInheritedTermNames(TermType type) {
         Set<String> result = new HashSet<String>();
         forAllTerms(type, (k,v) -> {
-            if (v.isCopiedFromDefault()) result.add(k);            
-        });                
+            if (v.isCopiedFromDefault()) result.add(k);
+        });
         return result;
     }
 
@@ -669,12 +669,12 @@ public void writeJson(Appendable appendable, boolean useRenamedNouns, Collection
      * @throws IOException if an error happens during append
      */
     public void writeJsonTerms(Appendable out, boolean useRenamedNouns, Collection<GrammaticalTerm> terms) throws IOException {
-        Collection<String> termsToInclude = 
-                terms == null ? null : 
+        Collection<String> termsToInclude =
+                terms == null ? null :
                 terms.stream().map((t)->t.getName()).collect(Collectors.toSet());
         writeJson(out, useRenamedNouns, termsToInclude);
     }
-    
+
 
     private void writeObject(ObjectOutputStream out) throws IOException {
         makeSkinny();
@@ -686,7 +686,6 @@ protected void writeObjectInternal(ObjectOutputStream out) throws IOException {
         // do nothing here - for hook
     }
 
-    @SuppressWarnings("unchecked")
     private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
         in.defaultReadObject();
 
@@ -720,9 +719,9 @@ public Noun getNounOverride(Noun n) {
     }
 
     private static class TreeSetSupplier<V> implements Supplier<SortedSet<V>>, Serializable {
-		private static final long serialVersionUID = 1L;
+        private static final long serialVersionUID = 1L;
 
-		@Override
+        @Override
         public SortedSet<V> get() {
             return new TreeSet<V>();
         }

src/main/java/com/force/i18n/grammar/impl/ComplexGrammaticalForm.java

@@ -39,6 +39,7 @@
 import com.force.i18n.grammar.ModifierForm;
 import com.force.i18n.grammar.Noun;
 import com.force.i18n.grammar.NounForm;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Multimap;
@@ -137,7 +138,7 @@ protected Object readResolve() {
                 case Adjective: formList = declension.getAdjectiveForms(); break;
                 case Article: formList = declension.getArticleForms(); break;
             }
-            assert formList != null;
+            Preconditions.checkState(formList != null);
             return formList.get(this.ordinal);
         }
     }
@@ -273,7 +274,7 @@ static <T extends ComplexGrammaticalForm> void serializeFormMap(ObjectOutputStre
         } else {
             out.writeByte(values.size());
             for (Map.Entry<T,String> entry : values.entrySet()) {
-                out.writeByte(entry.getKey().getOrdinal());                
+                out.writeByte(entry.getKey().getOrdinal());
                 out.writeUTF(entry.getValue());  // Serialize the "object" because it's been uniquefied
             }
         }
@@ -292,7 +293,8 @@ static <T extends ComplexGrammaticalForm> Map<T, String> deserializeFormMap(Obje
         case Adjective: formList = (List<T>)declension.getAdjectiveForms(); break;
         case Article: formList = (List<T>)declension.getArticleForms(); break;
         }
-        assert formList != null;
+        Preconditions.checkState(formList != null);
+
         for (int i = 0; i < size; i++) {
             int ordinal = in.readByte();
             String value = intern(in.readUTF());

src/main/java/com/force/i18n/grammar/impl/EnglishDeclension.java

@@ -8,6 +8,7 @@
 package com.force.i18n.grammar.impl;
 
 import static com.force.i18n.commons.util.settings.IniFileUtil.intern;
+import static com.google.common.base.Preconditions.checkArgument;
 
 import java.io.IOException;
 import java.util.*;
@@ -25,18 +26,18 @@
  * @author stamm
  */
 class EnglishDeclension extends ArticledDeclension {
-	private static final Logger logger = Logger.getLogger(EnglishDeclension.class.getName());
+    private static final Logger logger = Logger.getLogger(EnglishDeclension.class.getName());
 
-	public EnglishDeclension(HumanLanguage language) {
-		super(language);
-        assert language.getLocale().getLanguage().equals("en") : "Initializing a language that isn't english";
-	}
+    public EnglishDeclension(HumanLanguage language) {
+        super(language);
+        checkArgument(language.getLocale().getLanguage().equals("en"), "Initializing a language that isn't english");
+    }
 
     /**
      * The english articles are distinguished by whether the next noun starts with a vowel
      * sound or not (although the, unlike a/an is spelled the same).
      */
-    public static enum EnglishArticleForm implements ArticleForm {
+    public enum EnglishArticleForm implements ArticleForm {
         SINGULAR(LanguageNumber.SINGULAR, LanguageStartsWith.CONSONANT),
         SINGULAR_V(LanguageNumber.SINGULAR, LanguageStartsWith.VOWEL),
         PLURAL(LanguageNumber.PLURAL, LanguageStartsWith.CONSONANT)
@@ -59,17 +60,17 @@ static EnglishArticleForm getForm(ModifierForm form) {
                     (form.getStartsWith() == LanguageStartsWith.VOWEL ? SINGULAR_V : SINGULAR)
                     : PLURAL;
         }
-		@Override
-		public String getKey() {
-			return getNumber().getDbValue() + "-" + getStartsWith().getDbValue();
-		}
-
-		@Override
-		public void appendJsFormReplacement(Appendable a, String termFormVar, String genderVar, String startsWithVar)
-				throws IOException {
-			// The only variant is in the signular, so we don't want to screw up "the"
-			a.append(termFormVar+".charAt(0)=='"+LanguageNumber.PLURAL.getDbValue()+"'?"+termFormVar+":'"+LanguageNumber.SINGULAR.getDbValue()+"-'+"+startsWithVar);
-		}
+        @Override
+        public String getKey() {
+            return getNumber().getDbValue() + "-" + getStartsWith().getDbValue();
+        }
+
+        @Override
+        public void appendJsFormReplacement(Appendable a, String termFormVar, String genderVar, String startsWithVar)
+                throws IOException {
+            // The only variant is in the signular, so we don't want to screw up "the"
+            a.append(termFormVar+".charAt(0)=='"+LanguageNumber.PLURAL.getDbValue()+"'?"+termFormVar+":'"+LanguageNumber.SINGULAR.getDbValue()+"-'+"+startsWithVar);
+        }
     }
 
     /**