Skip to content

Commit c778535

Browse files
tisonkuntisonkun
committed
ci: download snapshots from datasketches-tck
1 parent 72b04b0 commit c778535

2 files changed

Lines changed: 165 additions & 31 deletions

File tree

‎.github/workflows/cpp-serde-compat.yml‎

Lines changed: 16 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: CPP SerDe Compatibility Test
1+
name: SerDe Compatibility Test
22

33
on:
44
push:
@@ -12,45 +12,30 @@ on:
1212

1313
jobs:
1414
build:
15-
name: SerDe Test
15+
name: ${{ matrix.name }} SerDe Test
1616
runs-on: ubuntu-latest
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
include:
21+
- language: cpp
22+
name: C++
23+
profile: check-cpp-files
24+
- language: go
25+
name: Go
26+
profile: check-go-files
1727
steps:
1828
- name: Checkout
1929
uses: actions/checkout@v5
2030

21-
- name: Checkout C++
22-
uses: actions/checkout@v5
23-
with:
24-
repository: apache/datasketches-cpp
25-
path: cpp
26-
2731
- name: Setup Java
2832
uses: actions/setup-java@v5
2933
with:
3034
java-version: '25'
3135
distribution: 'temurin'
3236

33-
- name: Configure C++ build
34-
run: cd cpp/build && cmake .. -DGENERATE=true
35-
36-
- name: Build C++ unit tests
37-
run: cd cpp && cmake --build build --config Release
38-
39-
- name: Run C++ tests
40-
run: cd cpp && cmake --build build --config Release --target test
37+
- name: Download ${{ matrix.name }} snapshots
38+
run: python3 tools/download_serialization_test_data.py ${{ matrix.language }}
4139

42-
- name: Make dir
43-
run: mkdir -p serialization_test_data/cpp_generated_files
44-
45-
- name: Copy files
46-
run: cp cpp/build/*/test/*_cpp.sk serialization_test_data/cpp_generated_files
47-
48-
- name: Run Java tests
49-
run: mvn test -P check-cpp-files
50-
51-
- name: Upload C++ Generated Sketch Files
52-
uses: actions/upload-artifact@v7
53-
with:
54-
name: cpp_generated_files
55-
path: serialization_test_data/cpp_generated_files/
56-
retention-days: 30
40+
- name: Run Java tests against ${{ matrix.name }} snapshots
41+
run: mvn test -P ${{ matrix.profile }}

‎tools/download_serialization_test_data.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/usr/bin/env python3
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
import argparse
21+
import shutil
22+
import tarfile
23+
import tempfile
24+
import urllib.request
25+
from pathlib import Path, PurePosixPath
26+
27+
28+
# Pin the archive so compatibility tests always use an immutable snapshot set.
29+
TCK_REVISION = "d363b12d293b395d90abb42677f9ea63178dbc0d"
30+
TCK_ARCHIVE_URL = (
31+
f"https://api.github.com/repos/apache/datasketches-tck/tarball/{TCK_REVISION}"
32+
)
33+
SUPPORTED_LANGUAGES = ("cpp", "go")
34+
35+
36+
def download_archive(destination: Path) -> None:
37+
print(f"Downloading serialization snapshots from {TCK_ARCHIVE_URL}", flush=True)
38+
request = urllib.request.Request(
39+
TCK_ARCHIVE_URL,
40+
headers={
41+
"Accept": "application/vnd.github+json",
42+
"User-Agent": "apache-datasketches-java",
43+
"X-GitHub-Api-Version": "2022-11-28",
44+
},
45+
)
46+
with urllib.request.urlopen(request, timeout=60) as response:
47+
with destination.open("wb") as output:
48+
shutil.copyfileobj(response, output)
49+
50+
51+
def extract_snapshots(archive_path: Path, languages: tuple[str, ...]) -> None:
52+
repository_root = Path(__file__).resolve().parents[1]
53+
serialization_data = repository_root / "serialization_test_data"
54+
serialization_data.mkdir(parents=True, exist_ok=True)
55+
56+
staging_directories = {
57+
language: Path(
58+
tempfile.mkdtemp(
59+
prefix=f".{language}_generated_files-",
60+
dir=serialization_data,
61+
)
62+
)
63+
for language in languages
64+
}
65+
counts = dict.fromkeys(languages, 0)
66+
67+
try:
68+
with tarfile.open(archive_path, mode="r:gz") as archive:
69+
for member in archive:
70+
if not member.isfile():
71+
continue
72+
73+
path = PurePosixPath(member.name)
74+
if path.suffix != ".sk":
75+
continue
76+
77+
language = next(
78+
(
79+
candidate
80+
for candidate in languages
81+
if path.parent.parts[-3:]
82+
== ("serialization", candidate, "snapshots")
83+
),
84+
None,
85+
)
86+
if language is None:
87+
continue
88+
89+
source = archive.extractfile(member)
90+
if source is None:
91+
raise RuntimeError(f"could not read snapshot from archive: {path}")
92+
93+
destination = staging_directories[language] / path.name
94+
if destination.exists():
95+
raise RuntimeError(f"duplicate snapshot in archive: {path.name}")
96+
with source, destination.open("wb") as output:
97+
shutil.copyfileobj(source, output)
98+
counts[language] += 1
99+
100+
for language, count in counts.items():
101+
if count == 0:
102+
raise RuntimeError(
103+
f"no {language} snapshots found in the TCK archive"
104+
)
105+
106+
for language, staging_directory in staging_directories.items():
107+
destination = serialization_data / f"{language}_generated_files"
108+
if destination.is_symlink():
109+
raise RuntimeError(
110+
f"snapshot output path cannot be a symbolic link: {destination}"
111+
)
112+
if destination.exists():
113+
if not destination.is_dir():
114+
raise RuntimeError(
115+
f"snapshot output path is not a directory: {destination}"
116+
)
117+
shutil.rmtree(destination)
118+
staging_directory.replace(destination)
119+
print(
120+
f"Extracted {counts[language]} {language} snapshots into {destination}"
121+
)
122+
finally:
123+
for staging_directory in staging_directories.values():
124+
if staging_directory.exists():
125+
shutil.rmtree(staging_directory)
126+
127+
128+
def main() -> None:
129+
parser = argparse.ArgumentParser(
130+
description="Download serialization snapshots from apache/datasketches-tck."
131+
)
132+
parser.add_argument(
133+
"languages",
134+
choices=SUPPORTED_LANGUAGES,
135+
metavar="LANG",
136+
nargs="*",
137+
help="languages to download (cpp and go by default)",
138+
)
139+
args = parser.parse_args()
140+
languages = tuple(dict.fromkeys(args.languages or SUPPORTED_LANGUAGES))
141+
142+
with tempfile.TemporaryDirectory(prefix="datasketches-tck-") as temp_directory:
143+
archive_path = Path(temp_directory) / "datasketches-tck.tar.gz"
144+
download_archive(archive_path)
145+
extract_snapshots(archive_path, languages)
146+
147+
148+
if __name__ == "__main__":
149+
main()

0 commit comments

Comments
 (0)