Skip to content

Commit 4107e56

Browse files
srl295srl295
committed
ICU-23473 Action for Automated CLDR Import
1 parent 7a05700 commit 4107e56

1 file changed

Lines changed: 281 additions & 0 deletions

File tree

‎.github/workflows/cldr-integration.yml‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
name: Update ICU from CLDR
2+
3+
env:
4+
ICU_DIR: ${{ github.workspace }}/icu
5+
CLDR_DIR: ${{ github.workspace }}/cldr
6+
ICU4C_DIR: ${{ github.workspace }}/icu/icu4c
7+
TOOLS_ROOT: ${{ github.workspace }}/icu/tools
8+
ICU4J_DIR: ${{ github.workspace }}/icu/icu4j
9+
CLDR_DATA_DIR: ${{ github.workspace }}/cldr-prod
10+
JDK_VERSION: 21
11+
JDK_DISTRO: temurin
12+
13+
on:
14+
workflow_dispatch:
15+
inputs:
16+
cldr-ref:
17+
description: CLDR ref to build
18+
required: true
19+
default: 'main'
20+
cldr-repo:
21+
description: CLDR repo to build
22+
required: true
23+
default: 'unicode-org/cldr'
24+
icu-repo:
25+
description: ICU repo
26+
required: true
27+
default: 'unicode-org/icu'
28+
icu-ref:
29+
description: ICU branch/tag
30+
required: true
31+
default: 'main'
32+
jobs:
33+
proddata:
34+
# TODO CLDR-15060: This is ripe for caching
35+
name: Build Production Data
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: Clone CLDR
39+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
40+
with:
41+
ref: ${{ github.event.inputs.cldr-ref }}
42+
lfs: false
43+
repository: ${{ github.event.inputs.cldr-repo }}
44+
persist-credentials: false
45+
- name: Set up JDK
46+
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
47+
with:
48+
java-version: ${{ env.JDK_VERSION }}
49+
distribution: ${{ env.JDK_DISTRO }}
50+
- name: Cache local Maven repository
51+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
52+
with:
53+
path: ~/.m2/repository
54+
key: ${{ runner.os }}-maven-${{ hashFiles('tools/**/pom.xml') }}
55+
restore-keys: |
56+
${{ runner.os }}-maven-
57+
- name: Build with Maven
58+
run: >
59+
mvn -s .github/workflows/mvn-settings.xml -B compile install package --file tools/pom.xml
60+
-DskipTests=true -pl cldr-code
61+
env:
62+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
63+
- name: Upload cldr-code
64+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.4.0
65+
with:
66+
name: cldr-code
67+
path: tools/cldr-code/target/cldr-code.jar
68+
- name: Build prod Data
69+
run: |
70+
rm -rf target/cldr-prod
71+
mkdir -p target/cldr-prod/common
72+
mvn -s .github/workflows/mvn-settings.xml -B -DCLDR_DIR=$(pwd) -DCLDR_GITHUB_ANNOTATIONS=true --file=tools/pom.xml -pl cldr-code \
73+
exec:java -Dexec.mainClass=org.unicode.cldr.tool.GenerateProductionData \
74+
-Dexec.args="-d target/cldr-prod/common/"
75+
(cd target/cldr-prod/ && zip -r ../cldr-prod.zip *)
76+
env:
77+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
78+
- name: Upload cldr-prod
79+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.4.0
80+
with:
81+
name: cldr-prod
82+
path: target/cldr-prod.zip
83+
genicudata:
84+
name: Generate ICU4C Data
85+
runs-on: ubuntu-latest
86+
permissions:
87+
contents: write
88+
pull-requests: write
89+
needs: proddata
90+
steps:
91+
- name: Download CLDR Production Artifact
92+
id: downloadprod
93+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.1.8
94+
with:
95+
name: cldr-prod
96+
path: .
97+
- name: Download CLDR tools
98+
id: downloadtools
99+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.1.8
100+
with:
101+
name: cldr-code
102+
path: cldr-code
103+
- name: Set up JDK
104+
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
105+
with:
106+
java-version: ${{ env.JDK_VERSION }}
107+
distribution: ${{ env.JDK_DISTRO }}
108+
- name: Setup Maven Settings
109+
uses: s4u/maven-settings-action@60912582505985be4cc55d2b890eb32767f8de5f # v2.8.0
110+
with:
111+
servers: |
112+
[{
113+
"id": "github",
114+
"username": "${{ github.actor }}",
115+
"password": "${{ github.token }}"
116+
},{
117+
"id": "githubicu",
118+
"username": "${{ github.actor }}",
119+
"password": "${{ github.token }}"
120+
},{
121+
"id": "githubcldr",
122+
"username": "${{ github.actor }}",
123+
"password": "${{ github.token }}"
124+
}]
125+
- name: Create empty CLDR dir, unpack cldr-prod
126+
run: mkdir -p cldr cldr-prod && cd cldr-prod && unzip ../cldr-prod.zip
127+
- name: Clone CLDR
128+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
129+
with:
130+
ref: ${{ github.event.inputs.cldr-ref }}
131+
lfs: false
132+
repository: ${{ github.event.inputs.cldr-repo }}
133+
path: cldr
134+
persist-credentials: false
135+
- name: Clone ICU
136+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
137+
with:
138+
repository: ${{ github.event.inputs.icu-repo }}
139+
ref: ${{ github.event.inputs.icu-ref }}
140+
lfs: false # not needed here
141+
path: icu
142+
persist-credentials: false
143+
- name: Cache local Maven repository
144+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
145+
with:
146+
path: ~/.m2/repository
147+
key: ${{ runner.os }}-maven-${{ hashFiles('icu/tools/**/pom.xml') }}
148+
restore-keys: |
149+
${{ runner.os }}-maven-
150+
# TODO: update DTDs https://github.com/unicode-org/icu/blob/main/docs/processes/cldr-icu.md#3-make-pre-adjustments
151+
# TODO: update ICU4J build version and locales https://github.com/unicode-org/icu/blob/main/docs/processes/cldr-icu.md#3-make-pre-adjustments
152+
- name: Build and Install CLDR jar
153+
run: >
154+
env
155+
mvn --file=cldr/tools/pom.xml -B clean install -pl :cldr-all,:cldr-code -DskipTests -DskipITs
156+
- name: Generate ICU4C Data
157+
run: >
158+
env
159+
mvn -B --file=$TOOLS_ROOT/cldr/cldr-to-icu/pom.xml clean package -DskipTests -DskipITs &&
160+
cd $TOOLS_ROOT/cldr/cldr-to-icu && java -jar target/cldr-to-icu-1.0-SNAPSHOT-jar-with-dependencies.jar --cldrDataDir="$CLDR_DATA_DIR"
161+
env:
162+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
163+
- name: copy test data
164+
run: >
165+
cd $TOOLS_ROOT/cldr ; ant copy-cldr-testdata
166+
- name: Zip up icu-data.zip
167+
run: 'cd icu/icu4c/source && zip -rlq ../../../icu-data.zip data'
168+
- name: Upload icu-data.zip
169+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.4.0
170+
with:
171+
name: icu-data
172+
path: icu-data.zip
173+
- name: Update branch
174+
run: |
175+
cd $ICU4C_DIR
176+
git config user.name 'Steven R. Loomis [bot]'
177+
git config user.email 'srloomis@unicode.org'
178+
git add source/data
179+
git commit -am 'ICU-00000 integration from CLDR'
180+
git checkout -b srl295/auto-cldr-update
181+
git remote set-url origin "https://srl295:${GH_TOKEN}@github.com/srl295/icu.git"
182+
git push --set-upstream -f origin srl295/auto-cldr-update
183+
env:
184+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
185+
# clang release build with some options to enforce useful constraints.
186+
# Includes dependency checker on an in-source, optimized build.
187+
# Includes checking @draft etc. API tags vs. ifndef guards like
188+
# U_HIDE_DRAFT_API and U_FORCE_HIDE_DRAFT_API.
189+
# (FORCE guards make this tool pass but won't compile to working code.
190+
# See the testtagsguards.sh script for details.)
191+
icu4c-clang:
192+
name: ICU4C Build/Test, Ubuntu/Clang, new data
193+
runs-on: ubuntu-latest
194+
needs: genicudata
195+
steps:
196+
- name: Clone ICU
197+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
198+
with:
199+
repository: ${{ github.event.inputs.icu-repo }}
200+
ref: ${{ github.event.inputs.icu-ref }}
201+
lfs: false
202+
persist-credentials: false
203+
- name: Download icu-data.zip
204+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.1.8
205+
with:
206+
name: icu-data
207+
path: .
208+
- name: Unpack icu-data.zip
209+
run: cd icu4c/source && rm -rf data && unzip -o ../../icu-data.zip
210+
- name: ICU4C with clang
211+
env:
212+
CPPFLAGS: -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1
213+
CFLAGS: -Wimplicit-fallthrough
214+
CXXFLAGS: -Wimplicit-fallthrough
215+
run: |
216+
sudo apt-get -y install doxygen;
217+
cd icu4c/source;
218+
./runConfigureICU Linux;
219+
make -j 2;
220+
make dist
221+
- name: Upload icu4c
222+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.4.0
223+
with:
224+
name: icu4c
225+
path: icu4c/source/dist
226+
- name: ICU4C with clang test
227+
env:
228+
CPPFLAGS: -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1
229+
CFLAGS: -Wimplicit-fallthrough
230+
CXXFLAGS: -Wimplicit-fallthrough
231+
run: |
232+
cd icu4c/source;
233+
make -j 2 check;
234+
( cd test/depstest && python3 depstest.py ../../../source/ );
235+
( cd .. && source/test/hdrtst/testtagsguards.sh );
236+
icu4j-java16:
237+
name: ICU4J Build/Test, Java 16, new data
238+
runs-on: ubuntu-latest
239+
needs: genicudata
240+
steps:
241+
- name: Clone ICU
242+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
243+
with:
244+
repository: ${{ github.event.inputs.icu-repo }}
245+
ref: ${{ github.event.inputs.icu-ref }}
246+
lfs: true
247+
persist-credentials: false
248+
- name: Checkout lfs objects
249+
run: git lfs pull
250+
- name: Download icu-data.zip
251+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.1.8
252+
with:
253+
name: icu-data
254+
path: .
255+
- name: Unpack icu-data.zip
256+
run: cd icu4c/source && rm -rf data && unzip -o ../../icu-data.zip
257+
- name: Generate ICU4J data
258+
run: cd icu4c/source && ./runConfigureICU Linux && make -j2 && make -C data icu4j-data-install ICU4J_ROOT=$(cd ../../icu4j ; pwd)
259+
- uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
260+
with:
261+
java-version: ${{ env.JDK_VERSION }}
262+
distribution: ${{ env.JDK_DISTRO }}
263+
- name: ICU4J Build
264+
run: |
265+
cd icu4j;
266+
ant init;
267+
ant jar;
268+
ant releaseBinaries;
269+
- name: Upload icu4j
270+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.4.0
271+
with:
272+
name: icu4j
273+
path: icu4j/release
274+
- name: ICU4J Test
275+
run: |
276+
ant check;
277+
ant localespiCheck;
278+
- name: List failures (if any)
279+
run: |
280+
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
281+
if: ${{ failure() }}

0 commit comments

Comments
 (0)