CLDR-18312 json- workaround CLDRFile identity bug (#4373)

unicode-org · Feb 13, 2025 · e4e3025 · e4e3025
1 parent 2cd0409
commit e4e3025
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 9 deletions.
diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/json/Ldml2JsonConverter.java b/tools/cldr-code/src/main/java/org/unicode/cldr/json/Ldml2JsonConverter.java
@@ -596,10 +596,10 @@ private Map<JSONSection, List<CldrItem>> mapPathsToSections(
             if (cv > coverageValue) {
                 continue;
             }
-            // Discard root identity element unless the locale is root
-            // TODO: CLDR-17790 this code should not be needed.
+
+            // TODO: CLDR-17790 known issue - //ldml/identity inherits when it shouldn't.
             rootIdentityMatcher.reset(fullPath);
-            if (rootIdentityMatcher.matches() && !"root".equals(locID)) {
+            if (rootIdentityMatcher.matches() && !file.isHere(fullPath)) {
                 continue;
             }
 

diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/json/LdmlConvertRules.java b/tools/cldr-code/src/main/java/org/unicode/cldr/json/LdmlConvertRules.java
@@ -422,12 +422,8 @@ public static class SplittableAttributeSpec {
         "//ldml/numbers/otherNumberingSystems/finance"
     };
 
-    /**
-     * Root language id pattern should be discarded in all locales except root, even though the path
-     * will exist in a resolved CLDRFile.
-     */
-    public static final Pattern ROOT_IDENTITY_PATTERN =
-            Pattern.compile("//ldml/identity/language\\[@type=\"root\"\\]");
+    /** resolved identity should be discarded if inherited, known issue CLDR-17790 */
+    public static final Pattern ROOT_IDENTITY_PATTERN = Pattern.compile("//ldml/identity.*");
 
     /**
      * Version (coming from DTD) should be discarded everywhere. This information is now in

diff --git a/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestCLDRFile.java b/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestCLDRFile.java
@@ -1004,4 +1004,36 @@ public void testEnglishSideways() {
             }
         }
     }
+
+    public void TestInheritedIdentity() {
+        final Map<String, Set<String>> elementToPaths = new TreeMap<>();
+
+        final CLDRFile f = CLDRConfig.getInstance().getCLDRFile("hi_Latn_IN", true);
+        for (final String s : f.fullIterable()) {
+            if (s.startsWith("//ldml/identity")) {
+                final String element = XPathParts.getFrozenInstance(s).getElement(-1);
+                final Set<String> setForElement =
+                        elementToPaths.computeIfAbsent(element, (ignored) -> new TreeSet<>());
+                assertTrue(
+                        "Duplicate XPath: " + s + " in " + f.getLocaleID(), setForElement.add(s));
+                if (!f.isHere(s)) {
+                    // this path should not be inherited
+                    if (!logKnownIssue("CLDR-17790", "//ldml/identity has inherited paths")) {
+                        errln("Inherited path " + s + " in " + f.getLocaleID());
+                    }
+                }
+            }
+        }
+        for (final Set<String> set : elementToPaths.values()) {
+            if (set.size() > 1) {
+                if (!logKnownIssue(
+                        "CLDR-17790",
+                        "//ldml/identity has duplicate resolved paths for elements")) {
+                    errln(
+                            "Duplicate //ldml/identity paths: "
+                                    + String.join(" ", set.toArray(new String[0])));
+                }
+            }
+        }
+    }
 }