Skip to content

CLDR-18370 Expand datetime test data set by iterating over each test dimension #4665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

CLDR-18370 Expand datetime test data set by iterating over each test dimension #4665

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
83236ec
CLDR-18370 Refactor dimension set definitions to be global constants
echeran Apr 8, 2025
a1e2423
CLDR-18370 Add iteration over locales
echeran May 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 104 additions & 25 deletions tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateDateTimeTestData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
Expand All @@ -39,6 +40,7 @@
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.ICUServiceBuilder;
import org.unicode.cldr.util.Level;
import org.unicode.cldr.util.SimpleFactory.NoSourceDirectoryException;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.TempPrintWriter;

Expand Down Expand Up @@ -284,9 +286,15 @@ private static ZonedDateTime getZonedDateTimeFromTemporalDateInput(Map<Object, O
}

public static final Optional<CLDRFile> getCLDRFile(String locale) {
CLDRFile cldrFile =
CLDRFile cldrFile;

try {
cldrFile =
CLDR_FACTORY.make(
locale, true, DraftStatus.contributed); // don't include provisional data
locale, true, DraftStatus.contributed); // don't include provisional data
} catch (NoSourceDirectoryException nsde) {
return Optional.empty();
}

// This is the CLDR "effective coverage level"
Level coverageLevel =
Expand Down Expand Up @@ -1041,7 +1049,7 @@ public static String computeSkeletonFromSemanticSkeleton(
return sb.toString();
}

private static TestCase convertTestCaseInputToTestCase(
private static TestCase convertTestCaseInputToTestCaseForNonSemanticSkeleton(
ICUServiceBuilder icuServiceBuilder,
CLDRFile localeCldrFile,
TestCaseInput testCaseInput) {
Expand Down Expand Up @@ -1081,13 +1089,21 @@ private static TestCase convertTestCaseInputToTestCase(
return result;
}

/**
* Primary function for converting a {@code TestCaseInput} into a {@code TestCase}, with the
* logic to handle semantic skeleton inputs differently from non-semantic skeleton inputs.
* @param icuServiceBuilder
* @param localeCldrFile
* @param testCaseInput
* @return
*/
private static TestCase computeTestCase(
ICUServiceBuilder icuServiceBuilder,
CLDRFile localeCldrFile,
TestCaseInput testCaseInput) {

if (testCaseInput.fieldStyleCombo.semanticSkeleton == null) {
return convertTestCaseInputToTestCase(icuServiceBuilder, localeCldrFile, testCaseInput);
return convertTestCaseInputToTestCaseForNonSemanticSkeleton(icuServiceBuilder, localeCldrFile, testCaseInput);
} else {
String calendarStr = testCaseInput.calendar.getType();
String skeleton =
Expand Down Expand Up @@ -1117,32 +1133,38 @@ private static TestCase computeTestCase(
}
}

public static ImmutableSet<TestCase> getKernelTestCases() {
//-----------
// begin: manually defined dimensions of iteration value sets
//-----------

static List<Pair<ULocale, Calendar>> LOCALE_CALENDAR_PAIRS =
List.of(
Pair.of(ULocale.ENGLISH, new GregorianCalendar()),
Pair.of(ULocale.forLanguageTag("ar-SA"), new IslamicCalendar()),
Pair.of(ULocale.forLanguageTag("th-TH"), new BuddhistCalendar()),
Pair.of(ULocale.forLanguageTag("ja-JP"), new JapaneseCalendar()));

// more manually defined inputs
static List<LocalDateTime> DATE_TIMES =
List.of(
LocalDateTime.of(2000, 1, 1, 0, 0, 0),
LocalDateTime.of(2024, 7, 1, 8, 50, 7),
// Ramadan in Summer at 12:00 noon in the year 2014
LocalDateTime.of(2014, 7, 15, 12, 0, 0));

List<Pair<ULocale, Calendar>> LOCALE_CALENDAR_PAIRS =
List.of(
Pair.of(ULocale.ENGLISH, new GregorianCalendar()),
Pair.of(ULocale.forLanguageTag("ar-SA"), new IslamicCalendar()),
Pair.of(ULocale.forLanguageTag("th-TH"), new BuddhistCalendar()),
Pair.of(ULocale.forLanguageTag("ja-JP"), new JapaneseCalendar()));
static List<LocalDateTime> DATE_TIME_ONE_ONLY = List.of(DATE_TIMES.get(0));

List<LocalDateTime> DATE_TIMES =
List.of(
LocalDateTime.of(2000, 1, 1, 0, 0, 0),
LocalDateTime.of(2024, 7, 1, 8, 50, 7),
// Ramadan in Summer at 12:00 noon in the year 2014
LocalDateTime.of(2014, 7, 15, 12, 0, 0));
// TODO: add a 3rd time zone dynamically, which is the default time zone for the current
// locale in question when iterating over all locales
static List<TimeZone> STATIC_TIME_ZONES =
List.of(TimeZone.GMT_ZONE, TimeZone.getTimeZone("Australia/Adelaide"));

List<LocalDateTime> DATE_TIME_ONE_ONLY = List.of(DATE_TIMES.get(0));
static List<TimeZone> STATIC_TIME_ZONE_ONE_ONLY = List.of(STATIC_TIME_ZONES.get(0));

// TODO: add a 3rd time zone dynamically, which is the default time zone for the current
// locale in question when iterating over all locales
List<TimeZone> STATIC_TIME_ZONES =
List.of(TimeZone.GMT_ZONE, TimeZone.getTimeZone("Australia/Adelaide"));
//-----------
// end: manually defined dimensions of iteration value sets
//-----------

List<TimeZone> STATIC_TIME_ZONE_ONE_ONLY = List.of(STATIC_TIME_ZONES.get(0));
public static ImmutableSet<TestCase> getKernelTestCases() {

// setup of return value

Expand Down Expand Up @@ -1206,6 +1228,63 @@ public static ImmutableSet<TestCase> getKernelTestCases() {
return builder.build();
}

public static boolean isCLDRLocaleAtLeastModern(String localeStr) {
// This is the CLDR "effective coverage level"
Level coverageLevel =
CalculatedCoverageLevels.getInstance()
.getEffectiveCoverageLevel(localeStr);

return !(coverageLevel == null || !coverageLevel.isAtLeast(Level.MODERN));
}

public static ImmutableSet<TestCase> getAllTestCases() {
Set<TestCase> kernelSet = getKernelTestCases();

ImmutableSet.Builder<TestCase> newCasesBuilder = ImmutableSet.builder();

ICUServiceBuilder icuServiceBuilder = new ICUServiceBuilder();


// append kernel to result builder
newCasesBuilder.addAll(kernelSet);


// iterate over all locales, cross each locale with kernel set, append to result builder
Set<String> availableCLDRLanguages = CLDR_FACTORY.getAvailableLanguages();
availableCLDRLanguages.stream()
.filter(GenerateDateTimeTestData::isCLDRLocaleAtLeastModern)
.map(locStr -> new ULocale(locStr))
.sequential() // cannot go parallel when used shared object ICUServiceBuilder
.filter(Objects::nonNull)
.forEach(locale -> {
String locName = locale.getName();
CLDRFile cldrFile = getCLDRFile(locName).orElse(null);
if (cldrFile == null) {
return;
}
icuServiceBuilder.clearCache();
icuServiceBuilder.setCldrFile(cldrFile);
kernelSet.stream()
.map(testCase -> testCase.testCaseInput)
.map(testCaseInput -> {
// manually create a "deep copy" object to avoid unwanted mutation
TestCaseInput newInput = new TestCaseInput();
// override locale
testCaseInput.locale = locale;
// copy over all other fields
newInput.timeZone = testCaseInput.timeZone;
newInput.fieldStyleCombo = testCaseInput.fieldStyleCombo;
newInput.dateTime = testCaseInput.dateTime;
newInput.calendar = testCaseInput.calendar;
return newInput;
})
.map(testCaseInput -> computeTestCase(icuServiceBuilder, cldrFile, testCaseInput))
.forEach(newCasesBuilder::add);
});

return newCasesBuilder.build();
}

/**
* This struct class exists specifically to convert from the structured {@code TestCase} struct
* class to one that is appropriate for de-/serializing (formatting & parsing), ex: to give it a
Expand Down Expand Up @@ -1281,7 +1360,7 @@ public static void main(String[] args) throws IOException {
try (TempPrintWriter pw =
TempPrintWriter.openUTF8Writer(
CLDRPaths.TEST_DATA + OUTPUT_SUBDIR, OUTPUT_FILENAME)) {
ImmutableSet<TestCase> testCases = getKernelTestCases();
ImmutableSet<TestCase> testCases = getAllTestCases();
List<TestCaseSerde> testCaseSerdes =
testCases.stream()
.map(GenerateDateTimeTestData::convertTestCaseToSerialize)
Expand Down
Loading