Skip to content

Commit 12da246

Browse files
srl295srl295
committed
ICU-23473 Action for Automated CLDR Import
1 parent 7a05700 commit 12da246

1 file changed

Lines changed: 276 additions & 0 deletions

File tree

‎.github/workflows/cldr-integration.yml‎

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
name: Update ICU from CLDR
2+
3+
env:
4+
ICU_DIR: ${{ github.workspace }}/icu
5+
CLDR_DIR: ${{ github.workspace }}/cldr
6+
ICU4C_DIR: ${{ github.workspace }}/icu/icu4c
7+
TOOLS_ROOT: ${{ github.workspace }}/icu/tools
8+
ICU4J_DIR: ${{ github.workspace }}/icu/icu4j
9+
CLDR_DATA_DIR: ${{ github.workspace }}/cldr-prod
10+
JDK_VERSION: 21
11+
JDK_DISTRO: temurin
12+
13+
on:
14+
workflow_dispatch:
15+
inputs:
16+
cldr-ref:
17+
description: CLDR ref to build
18+
required: true
19+
default: 'main'
20+
cldr-repo:
21+
description: CLDR repo to build
22+
required: true
23+
default: 'unicode-org/cldr'
24+
icu-repo:
25+
description: ICU repo
26+
required: true
27+
default: 'unicode-org/icu'
28+
icu-ref:
29+
description: ICU branch/tag
30+
required: true
31+
default: 'main'
32+
jobs:
33+
proddata:
34+
# TODO CLDR-15060: This is ripe for caching
35+
name: Build Production Data
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: Clone CLDR
39+
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
40+
with:
41+
ref: ${{ github.event.inputs.cldr-ref }}
42+
lfs: false
43+
repository: ${{ github.event.inputs.cldr-repo }}
44+
- name: Set up JDK
45+
uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
46+
with:
47+
java-version: ${{ JDK_VERSION }}
48+
distribution: ${{ JDK_DISTRO }}
49+
- name: Cache local Maven repository
50+
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
51+
with:
52+
path: ~/.m2/repository
53+
key: ${{ runner.os }}-maven-${{ hashFiles('tools/**/pom.xml') }}
54+
restore-keys: |
55+
${{ runner.os }}-maven-
56+
- name: Build with Maven
57+
run: >
58+
mvn -s .github/workflows/mvn-settings.xml -B compile install package --file tools/pom.xml
59+
-DskipTests=true -pl cldr-code
60+
env:
61+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
62+
- name: Upload cldr-code
63+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
64+
with:
65+
name: cldr-code
66+
path: tools/cldr-code/target/cldr-code.jar
67+
- name: Build prod Data
68+
run: |
69+
rm -rf target/cldr-prod
70+
mkdir -p target/cldr-prod/common
71+
mvn -s .github/workflows/mvn-settings.xml -B -DCLDR_DIR=$(pwd) -DCLDR_GITHUB_ANNOTATIONS=true --file=tools/pom.xml -pl cldr-code \
72+
exec:java -Dexec.mainClass=org.unicode.cldr.tool.GenerateProductionData \
73+
-Dexec.args="-d target/cldr-prod/common/"
74+
(cd target/cldr-prod/ && zip -r ../cldr-prod.zip *)
75+
env:
76+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
77+
- name: Upload cldr-prod
78+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
79+
with:
80+
name: cldr-prod
81+
path: target/cldr-prod.zip
82+
genicudata:
83+
name: Generate ICU4C Data
84+
runs-on: ubuntu-latest
85+
permissions:
86+
contents: write
87+
pull-requests: write
88+
needs: proddata
89+
steps:
90+
- name: Download CLDR Production Artifact
91+
id: downloadprod
92+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
93+
with:
94+
name: cldr-prod
95+
path: .
96+
- name: Download CLDR tools
97+
id: downloadtools
98+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
99+
with:
100+
name: cldr-code
101+
path: cldr-code
102+
- name: Set up JDK
103+
uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
104+
with:
105+
java-version: ${{ JDK_VERSION }}
106+
distribution: ${{ JDK_DISTRO }}
107+
- name: Setup Maven Settings
108+
uses: s4u/maven-settings-action@60912582505985be4cc55d2b890eb32767f8de5f # v2.8.0
109+
with:
110+
servers: |
111+
[{
112+
"id": "github",
113+
"username": "${{ github.actor }}",
114+
"password": "${{ github.token }}"
115+
},{
116+
"id": "githubicu",
117+
"username": "${{ github.actor }}",
118+
"password": "${{ github.token }}"
119+
},{
120+
"id": "githubcldr",
121+
"username": "${{ github.actor }}",
122+
"password": "${{ github.token }}"
123+
}]
124+
- name: Create empty CLDR dir, unpack cldr-prod
125+
run: mkdir -p cldr cldr-prod && cd cldr-prod && unzip ../cldr-prod.zip
126+
- name: Clone CLDR
127+
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
128+
with:
129+
ref: ${{ github.event.inputs.cldr-ref }}
130+
lfs: false
131+
repository: ${{ github.event.inputs.cldr-repo }}
132+
path: cldr
133+
- name: Clone ICU
134+
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
135+
with:
136+
repository: ${{ github.event.inputs.icu-repo }}
137+
ref: ${{ github.event.inputs.icu-ref }}
138+
lfs: false # not needed here
139+
path: icu
140+
- name: Cache local Maven repository
141+
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
142+
with:
143+
path: ~/.m2/repository
144+
key: ${{ runner.os }}-maven-${{ hashFiles('icu/tools/**/pom.xml') }}
145+
restore-keys: |
146+
${{ runner.os }}-maven-
147+
# TODO: update DTDs https://github.com/unicode-org/icu/blob/main/docs/processes/cldr-icu.md#3-make-pre-adjustments
148+
# TODO: update ICU4J build version and locales https://github.com/unicode-org/icu/blob/main/docs/processes/cldr-icu.md#3-make-pre-adjustments
149+
- name: Build and Install CLDR jar
150+
run: >
151+
env
152+
mvn --file=cldr/tools/pom.xml -B clean install -pl :cldr-all,:cldr-code -DskipTests -DskipITs
153+
- name: Generate ICU4C Data
154+
run: >
155+
env
156+
mvn -B --file=$TOOLS_ROOT/cldr/cldr-to-icu/pom.xml clean package -DskipTests -DskipITs &&
157+
cd $TOOLS_ROOT/cldr/cldr-to-icu && java -jar target/cldr-to-icu-1.0-SNAPSHOT-jar-with-dependencies.jar --cldrDataDir="$CLDR_DATA_DIR"
158+
env:
159+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
160+
- name: copy test data
161+
run: >
162+
cd $TOOLS_ROOT/cldr ; ant copy-cldr-testdata
163+
- name: Zip up icu-data.zip
164+
run: 'cd icu/icu4c/source && zip -rlq ../../../icu-data.zip data'
165+
- name: Upload icu-data.zip
166+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
167+
with:
168+
name: icu-data
169+
path: icu-data.zip
170+
- name: Update branch
171+
run: |
172+
cd $ICU4C_DIR
173+
git config user.name 'Steven R. Loomis [bot]'
174+
git config user.email 'srloomis@unicode.org'
175+
git add source/data
176+
git commit -am 'ICU-00000 integration from CLDR'
177+
git checkout -b srl295/auto-cldr-update
178+
git remote set-url origin "https://srl295:${GH_TOKEN}@github.com/srl295/icu.git"
179+
git push --set-upstream -f origin srl295/auto-cldr-update
180+
env:
181+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
182+
# clang release build with some options to enforce useful constraints.
183+
# Includes dependency checker on an in-source, optimized build.
184+
# Includes checking @draft etc. API tags vs. ifndef guards like
185+
# U_HIDE_DRAFT_API and U_FORCE_HIDE_DRAFT_API.
186+
# (FORCE guards make this tool pass but won't compile to working code.
187+
# See the testtagsguards.sh script for details.)
188+
icu4c-clang:
189+
name: ICU4C Build/Test, Ubuntu/Clang, new data
190+
runs-on: ubuntu-latest
191+
needs: genicudata
192+
steps:
193+
- name: Clone ICU
194+
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
195+
with:
196+
repository: ${{ github.event.inputs.icu-repo }}
197+
ref: ${{ github.event.inputs.icu-ref }}
198+
lfs: false
199+
- name: Download icu-data.zip
200+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
201+
with:
202+
name: icu-data
203+
path: .
204+
- name: Unpack icu-data.zip
205+
run: cd icu4c/source && rm -rf data && unzip -o ../../icu-data.zip
206+
- name: ICU4C with clang
207+
env:
208+
CPPFLAGS: -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1
209+
CFLAGS: -Wimplicit-fallthrough
210+
CXXFLAGS: -Wimplicit-fallthrough
211+
run: |
212+
sudo apt-get -y install doxygen;
213+
cd icu4c/source;
214+
./runConfigureICU Linux;
215+
make -j 2;
216+
make dist
217+
- name: Upload icu4c
218+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
219+
with:
220+
name: icu4c
221+
path: icu4c/source/dist
222+
- name: ICU4C with clang test
223+
env:
224+
CPPFLAGS: -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1
225+
CFLAGS: -Wimplicit-fallthrough
226+
CXXFLAGS: -Wimplicit-fallthrough
227+
run: |
228+
cd icu4c/source;
229+
make -j 2 check;
230+
( cd test/depstest && python3 depstest.py ../../../source/ );
231+
( cd .. && source/test/hdrtst/testtagsguards.sh );
232+
icu4j-java:
233+
name: ICU4J Build/Test, Java, new data
234+
runs-on: ubuntu-latest
235+
needs: genicudata
236+
steps:
237+
- name: Clone ICU
238+
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
239+
with:
240+
repository: ${{ github.event.inputs.icu-repo }}
241+
ref: ${{ github.event.inputs.icu-ref }}
242+
lfs: true
243+
- name: Checkout lfs objects
244+
run: git lfs pull
245+
- name: Download icu-data.zip
246+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
247+
with:
248+
name: icu-data
249+
path: .
250+
- name: Unpack icu-data.zip
251+
run: cd icu4c/source && rm -rf data && unzip -o ../../icu-data.zip
252+
- name: Generate ICU4J data
253+
run: cd icu4c/source && ./runConfigureICU Linux && make -j2 && make -C data icu4j-data-install ICU4J_ROOT=$(cd ../../icu4j ; pwd)
254+
- uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
255+
with:
256+
java-version: ${{ JDK_VERSION }}
257+
distribution: ${{ JDK_DISTRO }}
258+
- name: ICU4J Build
259+
run: |
260+
cd icu4j;
261+
ant init;
262+
ant jar;
263+
ant releaseBinaries;
264+
- name: Upload icu4j
265+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
266+
with:
267+
name: icu4j
268+
path: icu4j/release
269+
- name: ICU4J Test
270+
run: |
271+
ant check;
272+
ant localespiCheck;
273+
- name: List failures (if any)
274+
run: |
275+
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
276+
if: ${{ failure() }}

0 commit comments

Comments
 (0)