feat(dir): add integration tests for server

.github/workflows/reusable-test-integration.yaml

@@ -0,0 +1,62 @@
+# Copyright AGNTCY Contributors (https://github.com/agntcy)
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test Integration
+
+on:
+  workflow_call:
+    inputs:
+      enable_coverage:
+        required: false
+        type: boolean
+        default: false
+        description: "Whether to collect and upload coverage as artifacts."
+
+jobs:
+  integration:
+    name: Test ${{ matrix.label }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - task: test:integration:server
+            label: "Server"
+            coverage_path: ".coverage/integration/server.out"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
+        with:
+          registry: ghcr.io
+          username: notused
+          password: ${{ secrets.GITHUB_TOKEN  }}
+
+      - name: Install Task
+        uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 #v2.0.0
+
+      - name: Setup Cosign
+        uses: sigstore/cosign-installer@ba7bc0a3fef59531c69a25acd34668d6d3fe6f22 # v4.1.0
+
+      - name: Setup Go
+        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        with:
+          go-version: "1.26.1"
+          cache-dependency-path: "**/*.sum"
+          cache: true
+
+      - name: Run tests
+        run: |
+          COVERAGE_ENABLED="${{ inputs.enable_coverage }}" task ${{ matrix.task }}
+
+      - name: Upload coverage artifact
+        if: ${{ inputs.enable_coverage }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: coverage-integration-${{ matrix.label }}
+          path: ${{ matrix.coverage_path }}
+          include-hidden-files: true
+          retention-days: 1

.github/workflows/reusable-test.yaml

@@ -54,6 +54,13 @@ jobs:
       - name: Run unit tests
         run: |
           task test:unit
+
+  integration:
+    name: Integration
+    uses: ./.github/workflows/reusable-test-integration.yaml
+    with:
+      enable_coverage: true
+
   sdk:
     name: SDK
     uses: ./.github/workflows/reusable-test-sdk.yaml

Taskfile.deps.yml

@@ -330,3 +330,12 @@ tasks:
       - python3 {{.PRE_COMMIT_PYZ}} install
     status:
       - test -f {{.PRE_COMMIT_PYZ}}
+
+  deps:ginkgo:
+    desc: Install ginkgo
+    run: once
+    cmds:
+      - GOBIN="{{ .BIN_DIR }}" go install github.com/onsi/ginkgo/v2/ginkgo@v{{ .GINKGO_VERSION }}
+      - mv "{{ .BIN_DIR }}/ginkgo" "{{ .GINKGO_BIN }}"
+    status:
+      - test -f "{{ .GINKGO_BIN }}"

Taskfile.vars.yml

@@ -81,6 +81,8 @@ vars:
   TRIVY_BIN: "{{ .BIN_DIR }}/trivy-{{.TRIVY_VERSION}}"
   PRE_COMMIT_VERSION: "4.5.1"
   PRE_COMMIT_PYZ: "{{ .BIN_DIR }}/pre-commit-{{.PRE_COMMIT_VERSION}}.pyz"
+  GINKGO_VERSION: "2.28.1"
+  GINKGO_BIN: "{{ .BIN_DIR }}/ginkgo-{{ .GINKGO_VERSION }}"
 
   ## Coverage related values
   COVERAGE_DIR: '{{ .COVERAGE_DIR | default (print .ROOT_DIR "/.coverage") }}'

cli/cmd/daemon/start.go

@@ -62,7 +62,7 @@ func runStart(cmd *cobra.Command, _ []string) error {
 	reconcilerCfg := buildReconcilerConfig()
 
 	// Start the gRPC server.
-	srv, err := server.New(ctx, serverCfg)
+	srv, err := server.New(ctx, serverCfg, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create server: %w", err)
 	}

server/database/database.go

@@ -100,6 +100,20 @@ func newSQLite(cfg config.SQLiteConfig) (*gormdb.DB, error) {
 
 // newPostgres creates a new database connection using PostgreSQL driver.
 func newPostgres(cfg config.PostgresConfig) (*gormdb.DB, error) {
+	db, err := NewPostgresGormDb(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to PostgreSQL database: %w", err)
+	}
+
+	gdb, err := gormdb.New(db)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize PostgreSQL database: %w", err)
+	}
+
+	return gdb, nil
+}
+
+func NewPostgresGormDb(cfg config.PostgresConfig) (*gorm.DB, error) {
 	host := cfg.Host
 	if host == "" {
 		host = config.DefaultPostgresHost
@@ -123,17 +137,7 @@ func newPostgres(cfg config.PostgresConfig) (*gormdb.DB, error) {
 	dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
 		host, port, cfg.Username, cfg.Password, database, sslMode)
 
-	db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
+	return gorm.Open(postgres.Open(dsn), &gorm.Config{ //nolint:wrapcheck
 		Logger: newCustomLogger(),
 	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to connect to PostgreSQL database: %w", err)
-	}
-
-	gdb, err := gormdb.New(db)
-	if err != nil {
-		return nil, fmt.Errorf("failed to initialize PostgreSQL database: %w", err)
-	}
-
-	return gdb, nil
 }

server/server.go

@@ -114,7 +114,7 @@ func buildConnectionOptions(cfg config.ConnectionConfig) []grpc.ServerOption {
 func Run(ctx context.Context, cfg *config.Config) error {
 	errCh := make(chan error)
 
-	server, err := New(ctx, cfg)
+	server, err := New(ctx, cfg, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create server: %w", err)
 	}
@@ -153,7 +153,7 @@ func configureOASFValidation(cfg *config.Config) error {
 }
 
 //nolint:cyclop // This function has been at the limit; refactoring is out of scope.
-func New(ctx context.Context, cfg *config.Config) (*Server, error) {
+func New(ctx context.Context, cfg *config.Config, databaseAPI types.DatabaseAPI) (*Server, error) {
 	logger.Debug("Creating server with config", "config", cfg, "version", version.String())
 
 	if err := configureOASFValidation(cfg); err != nil {
@@ -227,9 +227,11 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
 		return nil, fmt.Errorf("failed to create routing: %w", err)
 	}
 
-	databaseAPI, err := database.New(cfg.Database)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create database API: %w", err)
+	if databaseAPI == nil {
+		databaseAPI, err = database.New(cfg.Database)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create database API: %w", err)
+		}
 	}
 
 	// Create JWT authentication service if enabled
@@ -317,6 +319,8 @@ func New(ctx context.Context, cfg *config.Config) (*Server, error) {
 	}, nil
 }
 
+func (s Server) GrpcServer() *grpc.Server { return s.grpcServer }
+
 func (s Server) Options() types.APIOptions { return s.options }
 
 func (s Server) Store() types.StoreAPI { return s.store }

tests/Taskfile.yml

@@ -607,3 +607,26 @@ tasks:
         else
           echo "No pods directory found"
         fi
+
+  test:integration:server:
+    desc: Run integration tests for the API server
+    dir: "{{ .ROOT_DIR }}/tests/integration-server"
+    deps:
+      - task: deps:ginkgo
+    vars:
+      COVERAGE_ENABLED: '{{ .COVERAGE_ENABLED | default "false" }}'
+    cmds:
+      - defer: docker compose down -v
+      - docker compose up -d --build --wait
+      - |
+        if [ "{{ .COVERAGE_ENABLED }}" = "true" ]; then
+          mkdir -p "{{ .COVERAGE_DIR }}/integration"
+          {{ .GINKGO_BIN }} --randomize-all \
+                 -p \
+                 -covermode="atomic" \
+                 -coverpkg="github.com/agntcy/dir/api/...,github.com/agntcy/dir/server/..." \
+                 --output-dir="{{ .COVERAGE_DIR }}/integration" \
+                 -coverprofile="server.out"
+        else
+          {{ .GINKGO_BIN }} --randomize-all -p
+        fi

tests/go.mod

@@ -21,15 +21,21 @@ require (
 	github.com/agntcy/dir/api v1.1.0
 	github.com/agntcy/dir/cli v1.1.0
 	github.com/agntcy/dir/client v1.1.0
+	github.com/agntcy/dir/server v1.1.0
+	github.com/brianvoe/gofakeit/v7 v7.14.1
 	github.com/google/uuid v1.6.0
+	github.com/joho/godotenv v1.5.1
 	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c
 	github.com/onsi/ginkgo/v2 v2.28.1
 	github.com/onsi/gomega v1.39.1
+	github.com/opencontainers/image-spec v1.1.1
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.21.0
 	google.golang.org/grpc v1.79.3
 	google.golang.org/protobuf v1.36.11
+	gorm.io/gorm v1.31.1
+	oras.land/oras-go/v2 v2.6.0
 )
 
 require (
@@ -61,7 +67,6 @@ require (
 	github.com/agntcy/dir/importer v1.1.0 // indirect
 	github.com/agntcy/dir/mcp v1.1.0 // indirect
 	github.com/agntcy/dir/reconciler v1.1.0 // indirect
-	github.com/agntcy/dir/server v1.1.0 // indirect
 	github.com/agntcy/dir/utils v1.1.0 // indirect
 	github.com/agntcy/oasf-sdk/pkg v1.0.3 // indirect
 	github.com/alecthomas/chroma/v2 v2.23.1 // indirect
@@ -286,7 +291,6 @@ require (
 	github.com/oklog/ulid/v2 v2.1.1 // indirect
 	github.com/openai/openai-go/v3 v3.29.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pion/datachannel v1.6.0 // indirect
@@ -397,7 +401,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/postgres v1.6.0 // indirect
-	gorm.io/gorm v1.31.1 // indirect
 	k8s.io/api v0.35.3 // indirect
 	k8s.io/apimachinery v0.35.3 // indirect
 	k8s.io/client-go v0.35.3 // indirect
@@ -409,7 +412,6 @@ require (
 	modernc.org/mathutil v1.5.0 // indirect
 	modernc.org/memory v1.5.0 // indirect
 	modernc.org/sqlite v1.23.1 // indirect
-	oras.land/oras-go/v2 v2.6.0 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect

tests/go.sum

@@ -191,6 +191,8 @@ github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6
 github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
 github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
+github.com/brianvoe/gofakeit/v7 v7.14.1 h1:a7fe3fonbj0cW3wgl5VwIKfZtiH9C3cLnwcIXWT7sow=
+github.com/brianvoe/gofakeit/v7 v7.14.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
 github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
 github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw=
@@ -673,6 +675,8 @@ github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW
 github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs=
 github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
 github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

tests/integration-server/README.md

@@ -0,0 +1,25 @@
+# Integration tests for the API server
+
+This suite runs the integration tests for the API server
+
+**Integration tests vs E2E tests**
+
+An integration test is similar to an end-to-end test in that they
+both test the software with dependencies included (e.g. with Postgres, Zot running)
+
+The difference between an integration test and an end-to-end test is that:
+- E2E tests use black-box testing to test workflows (higher level)
+- Integration use white-box testing to test side effects (lower level)
+
+E2E tests are higher level than integration tests:
+- E2E tests may include more dependencies, the setup may be more complex
+- Integration tests may include less dependencies, the setup may be less complex
+
+Ideally, integration tests are more isolated than E2E tests
+
+For example:
+- Each test has their own OCI repository to avoid conflicts in Zot
+- Each test is executed in a database transaction to avoid conflicts in Postgres
+
+As a result, integration tests can be executed in parallel, which makes them
+faster than E2E tests

tests/integration-server/docker-compose.yml

@@ -0,0 +1,32 @@
+name: dir-integration-tests-for-server
+services:
+  zot:
+    image: ghcr.io/project-zot/zot:v2.1.15
+    ports:
+      - 5556:5000
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+  postgres:
+    image: docker.io/bitnami/postgresql@sha256:e8b68936d05ee665974430bb4765fedcdc5c9ba399e133e120e2deed77f54dbf
+    environment:
+      POSTGRES_USER: ditfs_user
+      POSTGRES_PASSWORD: ditfs_pass
+      POSTGRES_DB: ditfs_db
+    ports:
+      - 5433:5432
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+    healthcheck:
+      test: pg_isready -U ditfs_user -d ditfs_pass || exit 1
+      interval: 10s
+      retries: 60
+      start_period: 10s
+networks:
+  default:
+    name: dir-integration-tests-for-server-network

tests/integration-server/integration_server_suite_test.go

@@ -0,0 +1,26 @@
+// Copyright AGNTCY Contributors (https://github.com/agntcy)
+// SPDX-License-Identifier: Apache-2.0
+
+package integration_server
+
+import (
+	"testing"
+
+	"github.com/agntcy/dir/tests/test_utils"
+	"github.com/joho/godotenv"
+	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+)
+
+const envFile = "server.env"
+
+func TestIntegrationServer(t *testing.T) {
+	err := godotenv.Load(envFile)
+	if err != nil {
+		t.Errorf("failed to load %s: %s", envFile, err)
+	}
+
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	test_utils.InitGofakeit()
+	ginkgo.RunSpecs(t, "E2E Production (production-like) Test Suite")
+}