Generate stefz files for testing (#144)

Instead of storing stefz in git we generate them before tests.
The files are fairly large and we store large delta everytime the
format changes.

Files are now deleted from git. otlp2stef tool is used to generate
stefz files from OTLP .pb files that don't change.

We also use a script to check compatibility of a format.
The script checks out old version of otlp2stef, generates
files from that, check outs current version of code and
runs new test code on old files.

Author: tigrannajaryan

.github/workflows/build-and-test.yml

@@ -29,6 +29,9 @@ jobs:
 
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # this is needed for "make testcompat" in benchmarks to checkout a past commit
+          fetch-depth: 0
 
       - name: Go cache
         uses: actions/cache@v4
@@ -71,6 +74,11 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Generate test files
+        run: |
+          cd benchmarks
+          make gentestfiles
+
       - name: Set up JDK for running Gradle
         uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
         with:
@@ -118,6 +126,11 @@ jobs:
         with:
           cache-read-only: ${{ github.event_name == 'pull_request' }}
 
+      - name: Generate test files
+        run: |
+          cd benchmarks
+          make gentestfiles
+
       - name: Generate Java code from test schemas
         run: |
           cd stefgen/generator

benchmarks/makefile

@@ -21,15 +21,33 @@ build:
 	go build -o bin/stefbench cmd/stefbench/main.go
 	go build -o bin/otlp2stef cmd/otlp2stef/main.go
 
+.PHONY: testcompat
+testcompat: # Test compatibility of current implementation with old format files
+	# Generate old test files
+	cd scripts && ./genoldtestfiles.sh
+	# Run compatibility tests
+	go test -v -run TestCopy
+	# Restore/generate current test files
+	cd scripts && ./gentestfiles.sh
+
+.PHONY: gentestfiles
+gentestfiles:
+	cd scripts && ./gentestfiles.sh
+
 .PHONY: test
-test:
+test: gentestfiles
 	go test -v ./...
 
-benchmark:
+benchmark: gentestfiles
 	go test ./... -run notest -bench . -benchtime 100ms
 
+.PHONY: build-ci
+build-ci:
+	make testcompat
+	make all
+
 .PHONY: benchmark-ci
-benchmark-ci: bench-run bench-stat-diff bench-stat
+benchmark-ci: gentestfiles bench-run bench-stat-diff bench-stat
 
 .PHONY: bench-stat-cli
 bench-stat-cli:
@@ -50,8 +68,3 @@ bench-stat: bench-stat-cli
 .PHONY: update-charts
 update-charts:
 	UPDATE_BENCH_HTML=1 go test -run TestMetricsSize\|TestMetricsMultipart -bench \(alizeNative\|Pdata\)
-
-regen-testdata: build
-	bin/otlp2stef --compression zstd --input testdata/astronomy-otelmetrics.zst
-	bin/otlp2stef --compression zstd --input testdata/hipstershop-otelmetrics.zst
-	bin/otlp2stef --compression zstd --input testdata/hostandcollector-otelmetrics.zst

benchmarks/readwrite_test.go

@@ -28,7 +28,7 @@ func TestCopy(t *testing.T) {
 
 	for _, file := range files {
 
-		tefBytes, err := os.ReadFile("testdata/" + file)
+		tefBytes, err := os.ReadFile("testdata/generated/" + file)
 		require.NoError(t, err)
 
 		tefReader, err := oteltef.NewMetricsReader(bytes.NewBuffer(tefBytes))
@@ -77,7 +77,7 @@ func TestCopy(t *testing.T) {
 }
 
 func BenchmarkReadSTEF(b *testing.B) {
-	tefBytes, err := os.ReadFile("testdata/hipstershop-otelmetrics.stefz")
+	tefBytes, err := os.ReadFile("testdata/generated/hipstershop-otelmetrics.stefz")
 	require.NoError(b, err)
 
 	tefSrc, err := oteltef.NewMetricsReader(bytes.NewBuffer(tefBytes))
@@ -131,7 +131,7 @@ func BenchmarkReadSTEF(b *testing.B) {
 }
 
 func BenchmarkReadSTEFZ(b *testing.B) {
-	tefBytes, err := os.ReadFile("testdata/hipstershop-otelmetrics.stefz")
+	tefBytes, err := os.ReadFile("testdata/generated/hipstershop-otelmetrics.stefz")
 	require.NoError(b, err)
 
 	recCount := 0
@@ -158,7 +158,7 @@ func BenchmarkReadSTEFZ(b *testing.B) {
 }
 
 func BenchmarkReadSTEFZWriteSTEF(b *testing.B) {
-	tefBytes, err := os.ReadFile("testdata/hipstershop-otelmetrics.stefz")
+	tefBytes, err := os.ReadFile("testdata/generated/hipstershop-otelmetrics.stefz")
 	require.NoError(b, err)
 
 	recCount := 0

benchmarks/scripts/genoldtestfiles.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# This script generates .stefz files from .zst files using the otlp2stef tool
+# built from a specific base branch. It checks out the base branch, builds the tool,
+# converts the files, and then checks out the original branch again.
+
+set -euo pipefail
+
+# Remember the current branch or commit hash if in detached HEAD state.
+CUR_BRANCH=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD)
+echo "Remembering current branch/commit hash as $CUR_BRANCH"
+
+TMPDIR=$(mktemp -d)
+cp ./gentestfiles.sh "$TMPDIR/"
+
+# Checkout the base branch to compare to
+BASE_BRANCH=21f498a293dbdf6e29c75385ba0043dc0f5e4009
+git -c advice.detachedHead=false checkout $BASE_BRANCH
+
+# Convert/generate the files.
+$TMPDIR/gentestfiles.sh
+
+# Checkout the previously remembered branch
+git checkout "$CUR_BRANCH"
+

benchmarks/scripts/gentestfiles.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This script generates .stefz files from .zst files using the otlp2stef tool
+# Generated files are placed in testdata/generated/.
+
+set -euo pipefail
+
+# Build otlp2stef tool (see makefile for how to do it)
+echo
+echo "Building otlp2stef tool"
+go build -o ../bin/otlp2stef ../cmd/otlp2stef/main.go
+
+# Copy .zst files to a temp directory
+echo
+echo "Converting .zst files to .stefz format using otlp2stef tool"
+TMPDIR=$(mktemp -d)
+cp ../testdata/astronomy-otelmetrics.zst "$TMPDIR/"
+cp ../testdata/hipstershop-otelmetrics.zst "$TMPDIR/"
+cp ../testdata/hostandcollector-otelmetrics.zst "$TMPDIR/"
+
+# Use otlp2stef tool to convert these files to stefz format
+for f in astronomy-otelmetrics hipstershop-otelmetrics hostandcollector-otelmetrics; do
+  ../bin/otlp2stef \
+    -compression=zstd \
+    --input="$TMPDIR/$f.zst"
+done
+
+# Copy stefz files from temp directory into testdata directory, overwriting if they exist
+echo
+echo "Copying converted .stefz files to ../testdata/generated/ directory"
+mkdir -p "../testdata/generated/"
+cp "$TMPDIR/astronomy-otelmetrics.stefz" ../testdata/generated/
+cp "$TMPDIR/hipstershop-otelmetrics.stefz" ../testdata/generated/
+cp "$TMPDIR/hostandcollector-otelmetrics.stefz" ../testdata/generated/
+
+echo "Test files generated."

benchmarks/testdata/.gitignore

@@ -0,0 +1 @@
+generated

benchmarks/testdata/astronomy-otelmetrics.stefz

@@ -24,7 +24,7 @@
 @Threads(1)
 @State(Scope.Thread)
 public class STEF {
-    public static final String stefRefFile = "../benchmarks/testdata/hipstershop-otelmetrics.stefz";
+    public static final String stefRefFile = "../benchmarks/testdata/generated/hipstershop-otelmetrics.stefz";
     public static long stefRecordCount = 66816; // Number of records in stefRefFile
 
     public ByteBuffer stefData;