Skip to content

Commit 23f01dd

Browse files
yuchannsyuchanns
authored
ci(bindings/go): simplify and improve Go bindings test infrastructure (#6293)
* ci(bindings/go): simplify and improve Go bindings test infrastructure Refactor the Go bindings CI workflow to use a more streamlined approach: - Replace complex matrix build configuration with simplified workflow - Enhance Makefile with better organized targets and workspace management - Update GitHub actions to use unified build process - Simplify documentation by removing Windows-specific complex setup * ci(bindings/go): reduce diff
1 parent 062307d commit 23f01dd

File tree

4 files changed

+80
-174
lines changed

4 files changed

+80
-174
lines changed

‎.github/actions/test_behavior_binding_go/action.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,13 @@ runs:
4040
uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
4141
- name: Run Test Binding Go
4242
shell: bash
43-
working-directory: bindings/go/tests/behavior_tests
4443
run: |
45-
export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/opt/libffi/lib
46-
go test -v -run TestBehavior
44+
if [ "${{ runner.os }}" == "Windows" ]; then
45+
# https://github.com/ilammy/msvc-dev-cmd/issues/25
46+
rm /usr/bin/link.exe
47+
fi
48+
cp bindings/go/Makefile .
49+
make tests
4750
env:
4851
OPENDAL_TEST: ${{ inputs.service }}
4952
CGO_ENABLE: 0

‎.github/workflows/test_behavior_binding_go.yml‎

Lines changed: 10 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -28,35 +28,22 @@ on:
2828
type: string
2929

3030
jobs:
31-
set-build:
32-
runs-on: ubuntu-latest
33-
outputs:
34-
build: ${{ steps.set-matrix-build.outputs.build }}
35-
steps:
36-
- uses: actions/checkout@v4
37-
- id: set-matrix-build
38-
name: Setup Matrix Build
39-
run: |
40-
MATRIX=$(yq -o=json -I=0 '[.build[] | select(.os == "${{ inputs.os }}")]' .github/scripts/test_go_binding/matrix.yaml | sed 's/ //g')
41-
echo "Matrix:"
42-
echo "$MATRIX" | jq .
43-
echo "build=$MATRIX" >> $GITHUB_OUTPUT
44-
4531
test:
46-
needs: [set-build]
4732
name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
4833
runs-on: ${{ inputs.os }}
4934
strategy:
5035
fail-fast: false
5136
matrix:
5237
cases: ${{ fromJson(inputs.cases) }}
53-
build: ${{ fromJson(needs.set-build.outputs.build) }}
5438
steps:
5539
- uses: actions/checkout@v4
40+
- uses: actions/checkout@v4
41+
with:
42+
path: "opendal"
5643
- uses: actions/checkout@v4
5744
with:
5845
repository: "apache/opendal-go-services"
59-
path: "tools"
46+
path: "opendal-go-services"
6047
- name: Setup Rust toolchain
6148
uses: ./.github/actions/setup
6249
with:
@@ -74,115 +61,28 @@ jobs:
7461
with:
7562
connect-host: ${{ secrets.OP_CONNECT_HOST }}
7663
connect-token: ${{ secrets.OP_CONNECT_TOKEN }}
77-
- name: Setup Target (Linux/macOS)
78-
if: runner.os != 'Windows'
79-
env:
80-
TARGET: ${{ matrix.build.target }}
81-
run: rustup target add $TARGET
82-
- name: Setup Target (Windows)
83-
if: runner.os == 'Windows'
84-
env:
85-
TARGET: ${{ matrix.build.target }}
86-
run: |
87-
rustup target add $env:TARGET
8864
- uses: actions/setup-go@v5
8965
with:
9066
go-version: stable
9167
cache-dependency-path: |
9268
bindings/go/go.sum
9369
bindings/go/tests/behavior_tests/go.sum
9470
- name: Install dependencies (Linux)
95-
if: ${{ matrix.build.os == 'ubuntu-latest' }}
71+
if: runner.os == 'Linux'
9672
run: sudo apt install zstd
9773
- name: Install dependencies (macOS)
98-
if: ${{ matrix.build.os == 'macos-latest' }}
74+
if: runner.os == 'macOS'
9975
run: brew install zstd libffi
10076
- name: Install dependencies (Windows)
101-
if: ${{ matrix.build.os == 'windows-latest' }}
102-
uses: ilammy/msvc-dev-cmd@v1
103-
- name: Build C Binding (Linux/macOS)
104-
working-directory: bindings/c
105-
if: runner.os != 'Windows'
106-
env:
107-
VERSION: "latest"
108-
SERVICE: ${{ matrix.cases.service }}
109-
TARGET: ${{ matrix.build.target }}
110-
CC: ${{ matrix.build.cc }}
111-
OS: ${{ matrix.build.os }}
112-
run: |
113-
SERVICE_DASH="${SERVICE//_/-}"
114-
cargo build --target $TARGET --release --features "opendal/services-${SERVICE_DASH}"
115-
DIR=$GITHUB_WORKSPACE/libopendal_c_${VERSION}_${SERVICE}_$TARGET
116-
mkdir $DIR
117-
if [ ${OS} == 'ubuntu-latest' ]; then
118-
SO=so
119-
else
120-
SO=dylib
121-
fi
122-
zstd -19 ./target/${TARGET}/release/libopendal_c.${SO} -o $DIR/libopendal_c.${TARGET}.$SO.zst
123-
- name: Build C Binding (Windows)
124-
working-directory: bindings/c
12577
if: runner.os == 'Windows'
126-
env:
127-
VERSION: "latest"
128-
SERVICE: ${{ matrix.cases.service }}
129-
TARGET: ${{ matrix.build.target }}
130-
CC: ${{ matrix.build.cc }}
131-
run: |
132-
$service_dash = ${env:SERVICE} -replace '_', '-'
133-
cargo build --target $env:TARGET --release --features "opendal/services-$service_dash"
134-
$DIR="$env:GITHUB_WORKSPACE\libopendal_c_${env:VERSION}_${env:SERVICE}_${env:TARGET}"
135-
Rename-Item -Path "./target/$env:TARGET/release/opendal_c.dll" -NewName "libopendal_c.dll"
136-
New-Item -ItemType Directory -Force -Path $DIR
137-
zstd -19 "./target/${env:TARGET}/release/libopendal_c.dll" -o "$DIR/libopendal_c.${env:TARGET}.dll.zst"
138-
- name: Build Go Artifact
139-
working-directory: tools/internal/generate
140-
env:
141-
MATRIX: '{"build": [${{ toJson(matrix.build) }}], "service": ["${{ matrix.cases.service }}"]}'
142-
VERSION: "latest"
143-
run: |
144-
go run generate.go
145-
- name: Setup Go Workspace (Linux/macOS)
146-
env:
147-
SERVICE: ${{ matrix.cases.service }}
148-
working-directory: bindings/go/tests
149-
if: runner.os != 'Windows'
150-
run: |
151-
go work init
152-
go work use ..
153-
go work use ./behavior_tests
154-
go work use $GITHUB_WORKSPACE/$(echo $SERVICE | sed 's/-/_/g')
155-
- name: Setup Go Workspace (Windows)
78+
uses: ilammy/msvc-dev-cmd@v1
79+
- name: Setup Go Workspace
15680
env:
15781
SERVICE: ${{ matrix.cases.service }}
158-
working-directory: bindings/go/tests
159-
if: runner.os == 'Windows'
16082
run: |
16183
go work init
162-
go work use ..
163-
go work use ./behavior_tests
164-
go work use $env:GITHUB_WORKSPACE/$($env:SERVICE -replace '-','_')
165-
- name: Generate Test Scheme
166-
env:
167-
SERVICE: ${{ matrix.cases.service }}
168-
run: |
169-
python ./.github/scripts/test_go_binding/generate_test_scheme.py
170-
- name: Setup Go Module (Linux/macOS)
171-
env:
172-
SERVICE: ${{ matrix.cases.service }}
173-
working-directory: bindings/go/tests/behavior_tests
174-
if: runner.os != 'Windows'
175-
run: |
176-
go mod edit -replace=github.com/apache/opendal-go-services/${SERVICE}=$GITHUB_WORKSPACE/$SERVICE
177-
go mod tidy
178-
- name: Setup Go Module (Windows)
179-
env:
180-
SERVICE: ${{ matrix.cases.service }}
181-
working-directory: bindings/go/tests/behavior_tests
182-
if: runner.os == 'Windows'
183-
run: |
184-
go mod edit -replace=github.com/apache/opendal-go-services/$env:SERVICE=$env:GITHUB_WORKSPACE/$env:SERVICE
185-
go mod tidy
84+
go work use ./opendal/bindings/go
85+
go work use ./opendal/bindings/go/tests/behavior_tests
18686
- name: Test Core
18787
uses: ./.github/actions/test_behavior_binding_go
18888
with:

‎bindings/go/Makefile‎

Lines changed: 63 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,17 +32,11 @@ endif
3232
# Define file extensions for different OS
3333
ifeq ($(OS),linux)
3434
LIB_EXT := so
35-
endif
36-
ifeq ($(OS),darwin)
35+
else ifeq ($(OS),darwin)
3736
LIB_EXT := dylib
38-
endif
39-
ifeq ($(findstring mingw,$(OS)),mingw)
40-
OS := windows
37+
else
4138
LIB_EXT := dll
42-
endif
43-
ifeq ($(findstring msys,$(OS)),msys)
4439
OS := windows
45-
LIB_EXT := dll
4640
endif
4741

4842
# Check if OPENDAL_TEST environment variable is set
@@ -67,10 +61,13 @@ FINAL_LIB_FILE := $(GITHUB_WORKSPACE)/$(SERVICE_PKG)/libopendal_c.$(OS).$(ARCH).
6761
# Build C library only when source files change
6862
$(C_LIB_FILE): opendal/bindings/c/src/*.rs opendal/bindings/c/Cargo.toml
6963
@echo "Building C library for service: $(SERVICE)"
70-
cd opendal/bindings/c && cargo build --features "opendal/services-$(SERVICE)"
64+
cd opendal/bindings/c && cargo build --features "opendal/services-$(shell echo $(SERVICE) | tr '_' '-')"
65+
ifeq ($(OS),windows)
66+
@echo "Renaming opendal_c.$(LIB_EXT) to libopendal_c.$(LIB_EXT) on Windows"
67+
mv opendal/bindings/c/target/debug/opendal_c.$(LIB_EXT) opendal/bindings/c/target/debug/libopendal_c.$(LIB_EXT)
68+
endif
7169

72-
# Create compressed library and generate services in one step
73-
# This target depends on the final library file location, not intermediate
70+
# Create compressed library and generate Go services
7471
$(FINAL_LIB_FILE): $(C_LIB_FILE)
7572
@echo "Compressing library and generating services for $(SERVICE) on $(OS)..."
7673
@mkdir -p $(GITHUB_WORKSPACE)/libopendal_c_$(VERSION)_$(SERVICE)_$(OS)
@@ -84,8 +81,58 @@ $(FINAL_LIB_FILE): $(C_LIB_FILE)
8481
@echo "Cleaning up intermediate directory..."
8582
@rm -rf $(GITHUB_WORKSPACE)/libopendal_c_$(VERSION)_$(SERVICE)_$(OS)
8683

87-
# Generate services target now just depends on the final file
88-
generate-services: $(FINAL_LIB_FILE)
84+
# Setup Go workspace for the service
85+
setup-workspace-$(SERVICE):
86+
@echo "Setting up Go workspace for service: $(SERVICE)"
87+
go work use $(GITHUB_WORKSPACE)/$(SERVICE)
88+
@echo "Setting up Go module replacement for service: $(SERVICE)"
89+
cd opendal/bindings/go/tests/behavior_tests && \
90+
go mod edit -replace=github.com/apache/opendal-go-services/$(SERVICE)=$(GITHUB_WORKSPACE)/$(SERVICE) && \
91+
go mod tidy
92+
93+
define SCHEME_TEST_CONTENT
94+
/*
95+
* Licensed to the Apache Software Foundation (ASF) under one
96+
* or more contributor license agreements. See the NOTICE file
97+
* distributed with this work for additional information
98+
* regarding copyright ownership. The ASF licenses this file
99+
* to you under the Apache License, Version 2.0 (the
100+
* "License"); you may not use this file except in compliance
101+
* with the License. You may obtain a copy of the License at
102+
*
103+
* http://www.apache.org/licenses/LICENSE-2.0
104+
*
105+
* Unless required by applicable law or agreed to in writing,
106+
* software distributed under the License is distributed on an
107+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
108+
* KIND, either express or implied. See the License for the
109+
* specific language governing permissions and limitations
110+
* under the License.
111+
*/
112+
// generated by github.com/apache/opendal/bindings/go
113+
114+
package opendal_test
115+
116+
import (
117+
opendal "github.com/apache/opendal/bindings/go"
118+
"github.com/apache/opendal-go-services/$(SERVICE)"
119+
)
120+
121+
// Add more schemes for behavior tests here.
122+
var schemes = []opendal.Scheme{
123+
$(SERVICE).Scheme,
124+
}
125+
endef
126+
127+
export SCHEME_TEST_CONTENT
128+
# Generate scheme test file
129+
generate-scheme-$(SERVICE):
130+
@echo "Generating scheme test file..."
131+
@mkdir -p opendal/bindings/go/tests/behavior_tests
132+
@echo "$$SCHEME_TEST_CONTENT" > opendal/bindings/go/tests/behavior_tests/scheme_test.go
133+
134+
# Generate services target with all steps
135+
generate-services: $(FINAL_LIB_FILE) setup-workspace-$(SERVICE) generate-scheme-$(SERVICE)
89136

90137
# Run tests
91138
run-tests: generate-services
@@ -101,7 +148,10 @@ clean:
101148
@echo "Cleaning all generated files..."
102149
rm -rf opendal/bindings/c/target
103150
rm -rf opendal-go-services/*/libopendal_c.*
151+
rm -rf opendal-go-services/*/opendal_c.*
104152
rm -rf opendal-go-services/libopendal_c_*
153+
rm -rf opendal-go-services/opendal_c_*
154+
cd opendal/bindings/go/tests/behavior_tests && go clean -testcache
105155

106156
# Help target
107157
help:

‎bindings/go/README.md‎

Lines changed: 1 addition & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ We use `go workspace` to manage and build the dependencies. To set up the worksp
222222

223223
<details>
224224
<summary>
225-
For Linux and macOS
225+
For Linux, macOS and Windows (MSVC)
226226
</summary>
227227

228228
<br/>
@@ -236,9 +236,6 @@ git clone --depth 1 git@github.com:apache/opendal-go-services.git
236236
go work init
237237
go work use ./opendal/bindings/go
238238
go work use ./opendal/bindings/go/tests/behavior_tests
239-
# use the backend you want to test, e.g., fs or memory
240-
go work use ./opendal-go-services/fs
241-
go work use ./opendal-go-services/memory
242239

243240
cp opendal/bindings/go/Makefile .
244241

@@ -261,50 +258,6 @@ cd -
261258

262259
</details>
263260

264-
<details>
265-
<summary>
266-
For Windows
267-
</summary>
268-
269-
<br/>
270-
271-
```powershell
272-
New-Item -ItemType Directory -Path opendal_workspace
273-
Set-Location -Path opendal_workspace
274-
275-
git clone --depth 1 git@github.com:apache/opendal.git
276-
git clone --depth 1 git@github.com:apache/opendal-go-services.git
277-
278-
go work init
279-
go work use ./opendal/bindings/go
280-
go work use ./opendal/bindings/go/tests/behavior_tests
281-
# use the backend you want to test, e.g., fs or memory
282-
go work use ./opendal-go-services/fs
283-
go work use ./opendal-go-services/memory
284-
285-
@'
286-
# Check if OPENDAL_TEST is set\;if (-not $env:OPENDAL_TEST) {\; Write-Error "OPENDAL_TEST environment variable is not set"\; Write-Host "Please set OPENDAL_TEST to specify which backend to test (e.g., fs or memory)"\; exit 1\;}\;# Specify the backend to test\;Set-Item -Path Env:SERVICE -Value "$env:OPENDAL_TEST"\;# Get architecture\;$architecture = (Get-WmiObject Win32_OperatingSystem).OSArchitecture\;\;if ($architecture -like "*64*") {\; $ARCH = "x86_64"\;} else {\; $ARCH = "unknown" \;}\;\;# Build opendal\;Push-Location opendal/bindings/c\;cargo build --features opendal/services-$($env:OPENDAL_TEST)\;Pop-Location\;\;# Rename dll file\;Rename-Item opendal/bindings/c/target/debug/opendal_c.dll libopendal_c.dll\;\;# Set environment variables\;Set-Item -Path Env:GITHUB_WORKSPACE -Value "$PWD/opendal-go-services"\;Set-Item -Path Env:VERSION -Value "latest"\;Set-Item -Path Env:TARGET -Value "windows"\;Set-Item -Path Env:DIR -Value "$($env:GITHUB_WORKSPACE)/libopendal_c_$($env:VERSION)_$($env:SERVICE)_$($env:TARGET)"\;\;if (-not (Test-Path $env:DIR)) {\; New-Item -ItemType Directory -Path $env:DIR\;}\;\;# Compress with zstd\;zstd -19 opendal/bindings/c/target/debug/libopendal_c.dll -o "$($env:DIR)/libopendal_c.windows.dll.zst"\;\;Push-Location opendal-go-services/internal/generate\;go run generate.go\;Pop-Location\;# Remove Unnecessary files\;Remove-Item -Path $env:DIR -Recurse -Force\;# Set environment variables\;Set-Item -Path Env:MATRIX -Value '{"build": [{"target":"windows", "goos":"windows", "goarch": "amd64"}], "service": ["fs"]}'\;# Assume that libffi-8.dll is in the root of workspace directory\;Set-Item -Path Env:PATH -Value "$($env:PATH);$PWD"\;# Run tests\;go test ./opendal/bindings/go/tests/behavior_tests -v -run TestBehavior\;
287-
'@ -replace "\\;","`n" | Out-File -FilePath "MakeTest.ps1" -Encoding UTF8
288-
289-
Pop-Location
290-
```
291-
292-
To build and run tests, run the following commands:
293-
294-
```powershell
295-
Set-Location -Path opendal_workspace
296-
# specify the backend to test
297-
$env:OPENDAL_TEST = "fs"
298-
$env:OPENDAL_FS_ROOT = $env:TEMP
299-
300-
# build the C binding and run the tests
301-
.\MakeTest.ps1
302-
303-
Pop-Location
304-
```
305-
306-
</details>
307-
308261
## License and Trademarks
309262

310263
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

0 commit comments

Comments
 (0)