diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 00000000..12da8a0e --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,29 @@ +name: Integration Tests +on: + push: + branches: + - master + pull_request: + branches: + - "*" + +jobs: + integration-tests: + runs-on: 32gb-runner + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.23.2' + + - name: Download Go dependencies + working-directory: ./server + run: go mod download + + - name: Run Docker In Docker Container Tests + working-directory: ./server + run: go test -v ./tests -timeout 0 -run 'TestDinDIntegration' \ No newline at end of file diff --git a/.gitignore b/.gitignore index 382c9961..2f3152b9 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,11 @@ npm-debug* # ---------------------------- *.env *.env.* + +# ---------------------------- +# Test Files +# ---------------------------- +ui/test-results/ +ui/playwright-report/ +ui/blob-report/ +ui/tests/.auth/ diff --git a/api-contract.md b/api-contract.md index d703fc08..6f46b8c1 100644 --- a/api-contract.md +++ b/api-contract.md @@ -127,15 +127,17 @@ http://localhost:8000 - **Response**: ```json - { - "success": "boolean", - "message": "string", - "data": { - "type": "string", - "version": "string", - "config": "json" - } +{ + "success": boolean, + "message": "string", + "data": { + "connection_result": { + "message": "string", + "status": "string" + }, + "logs": "json" } +} ``` ### Create Source @@ -305,15 +307,17 @@ http://localhost:8000 - **Response**: ```json - { - "success": "boolean", - "message": "string", - "data": { - "type": "string", - "version": "string", - "config": "json" - } + { + "success": boolean, + "message": "string", + "data": { + "connection_result": { + "message": "string", + "status": "string" + }, + "logs": "json" } +} ``` ### Create Destination @@ -503,6 +507,7 @@ http://localhost:8000 "frequency": "string", "last_run_time": "timestamp", "last_run_state": "string", + "last_run_type": "string", "created_at": "timestamp", "updated_at": "timestamp", "created_by": "string", // username @@ -538,6 +543,7 @@ http://localhost:8000 }, "frequency": "string", "streams_config": "json", + "difference_streams": "string", "activate": "boolean" // send this to activate or deactivate job } ``` @@ -673,10 +679,11 @@ http://localhost:8000 "message": "string", "data": [ { - "id": "string", + "file_path": "string", "start_time": "timestamp", "runtime": "integer", - "status": "string" + "status": "string", + "job_type": "string" } ] } @@ -787,6 +794,52 @@ http://localhost:8000 } ``` + +### Clear destination for a Job +--- + +- **Endpoint**: `/api/v1/project/:projectid/jobs/:id/clear-destination` +- **Method**: POST +- **Description**: clears the destination data +- **Headers**: `Authorization: Bearer ` + +- **Response**: + + ```json + { + "success": "boolean", + "message": "string", + "data": { + "message": "string" + } + } + ``` + + ### Difference Streams +- **Endpoint**: `/api/v1/project/:projectid/jobs/:id/difference-streams` +- **Method**: POST +- **Description**: returns the stream difference bewtween the saved and the updated streams +- **Headers**: `Authorization: Bearer ` +- - **Request Body**: + + ```json + { + "updated_streams_config": "json" + } + ``` + +- **Response**: + + ```json + { + "success": "boolean", + "message": "string", + "data": { + "difference_streams": "json" + } + } + ``` + ## Error Responses All endpoints may return the following error responses: diff --git a/server/go.mod b/server/go.mod index d8196ea6..eb1b9ec9 100644 --- a/server/go.mod +++ b/server/go.mod @@ -5,16 +5,19 @@ go 1.24.2 require github.com/beego/beego/v2 v2.3.8 require ( + github.com/apache/spark-connect-go/v35 v35.0.0-20250317154112-ffd832059443 github.com/aws/aws-sdk-go-v2/config v1.29.17 github.com/aws/aws-sdk-go-v2/service/ecr v1.50.5 github.com/aws/aws-sdk-go-v2/service/kms v1.41.1 + github.com/docker/docker v28.3.3+incompatible github.com/go-playground/validator/v10 v10.27.0 github.com/lib/pq v1.10.9 github.com/oklog/ulid v1.3.1 github.com/spf13/viper v1.20.1 + github.com/testcontainers/testcontainers-go v0.39.0 go.temporal.io/sdk v1.34.0 - golang.org/x/crypto v0.39.0 - golang.org/x/mod v0.25.0 + golang.org/x/crypto v0.41.0 + golang.org/x/mod v0.26.0 ) require ( @@ -33,19 +36,73 @@ require ( ) require ( + cloud.google.com/go/compute/metadata v0.7.0 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/apache/arrow-go/v18 v18.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/go-errors/errors v1.5.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/rogpeppe/go-internal v1.13.1 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/sdk v1.37.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.8.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/tools v0.35.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/go-viper/mapstructure/v2 v2.3.0 // indirect @@ -53,12 +110,13 @@ require ( github.com/golang/mock v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/jmoiron/sqlx v1.4.0 github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/nexus-rpc/sdk-go v0.3.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect @@ -67,24 +125,24 @@ require ( github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/afero v1.14.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/stretchr/testify v1.10.0 // indirect + github.com/stretchr/testify v1.11.1 github.com/subosito/gotenv v1.6.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.temporal.io/api v1.46.0 go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.28.0 // indirect golang.org/x/time v0.8.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/grpc v1.73.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/server/go.sum b/server/go.sum index dc54dd4c..c55aca9b 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,7 +1,25 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/apache/arrow-go/v18 v18.2.0 h1:QhWqpgZMKfWOniGPhbUxrHohWnooGURqL2R2Gg4SO1Q= +github.com/apache/arrow-go/v18 v18.2.0/go.mod h1:Ic/01WSwGJWRrdAZcxjBZ5hbApNJ28K96jGYaxzzGUc= +github.com/apache/spark-connect-go/v35 v35.0.0-20250317154112-ffd832059443 h1:pA4aHBVygvcQZuXVSJg2kH3z0rZO3M/YJUyUuPX82ko= +github.com/apache/spark-connect-go/v35 v35.0.0-20250317154112-ffd832059443/go.mod h1:ODlxb8YN0y/JyS7h+vhz+afnQ+beSkYTqDHYtg2T6E8= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I= github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= @@ -37,14 +55,39 @@ github.com/beego/beego/v2 v2.3.8/go.mod h1:8vl9+RrXqvodrl9C8yivX1e6le6deCK6RWeq8 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -53,18 +96,25 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -78,6 +128,8 @@ github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqw github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -89,19 +141,32 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -114,20 +179,58 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU= github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/nexus-rpc/sdk-go v0.3.0 h1:Y3B0kLYbMhd4C2u00kcYajvmOrfozEtTV/nHSnV57jA= github.com/nexus-rpc/sdk-go v0.3.0/go.mod h1:TpfkM2Cw0Rlk9drGkoiSMpFqflKTiQLWUNyKJjF8mKQ= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -145,11 +248,15 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 h1:DAYUYH5869yV94zvCES9F51oYtN5oGlwjxJJz7ZCnik= github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -164,27 +271,47 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/testcontainers/testcontainers-go v0.39.0 h1:uCUJ5tA+fcxbFAB0uP3pIK3EJ2IjjDUHFSZ1H1UxAts= +github.com/testcontainers/testcontainers-go v0.39.0/go.mod h1:qmHpkG7H5uPf/EvOORKvS6EuDkBUPE3zpVGaH9NL7f8= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE= +go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0= go.temporal.io/api v1.46.0 h1:O1efPDB6O2B8uIeCDIa+3VZC7tZMvYsMZYQapSbHvCg= go.temporal.io/api v1.46.0/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= go.temporal.io/sdk v1.34.0 h1:VLg/h6ny7GvLFVoQPqz2NcC93V9yXboQwblkRvZ1cZE= @@ -200,9 +327,11 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -210,8 +339,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -221,33 +350,43 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -260,28 +399,34 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= -google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -292,5 +437,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/server/internal/constants/messages.go b/server/internal/constants/messages.go index d3e8ef05..a39e3522 100644 --- a/server/internal/constants/messages.go +++ b/server/internal/constants/messages.go @@ -36,6 +36,7 @@ var ( ErrFailedToCreate = errors.New("failed to create") ErrFailedToUpdate = errors.New("failed to update") ErrFailedToDelete = errors.New("failed to delete") + ErrInProgress = errors.New("in progress") ) // Error message formats diff --git a/server/internal/dto/requests.go b/server/internal/dto/requests.go index bc8b0b6a..52d660f9 100644 --- a/server/internal/dto/requests.go +++ b/server/internal/dto/requests.go @@ -96,14 +96,14 @@ type CreateJobRequest struct { StreamsConfig string `json:"streams_config" orm:"type(jsonb)" validate:"required"` Activate bool `json:"activate,omitempty"` } - type UpdateJobRequest struct { - Name string `json:"name" validate:"required"` - Source JobSourceConfig `json:"source" validate:"required"` - Destination JobDestinationConfig `json:"destination" validate:"required"` - Frequency string `json:"frequency" validate:"required"` - StreamsConfig string `json:"streams_config" orm:"type(jsonb)" validate:"required"` - Activate bool `json:"activate,omitempty"` + Name string `json:"name" validate:"required"` + Source JobSourceConfig `json:"source" validate:"required"` + Destination JobDestinationConfig `json:"destination" validate:"required"` + Frequency string `json:"frequency" validate:"required"` + StreamsConfig string `json:"streams_config" orm:"type(jsonb)" validate:"required"` + DifferenceStreams string `json:"difference_streams" validate:"required"` + Activate bool `json:"activate,omitempty"` } type JobTaskRequest struct { @@ -112,3 +112,7 @@ type JobTaskRequest struct { type JobStatusRequest struct { Activate bool `json:"activate"` } + +type StreamDifferenceRequest struct { + UpdatedStreamsConfig string `json:"updated_streams_config" validate:"required"` +} diff --git a/server/internal/dto/response.go b/server/internal/dto/response.go index 9199968a..65d36e1b 100644 --- a/server/internal/dto/response.go +++ b/server/internal/dto/response.go @@ -44,6 +44,16 @@ type CheckUniqueJobNameResponse struct { Unique bool `json:"unique"` } +// TestConnectionResponse +type TestConnectionResponse struct { + ConnectionResult map[string]interface{} `json:"connection_result"` + Logs []map[string]interface{} `json:"logs"` +} + +type StreamDifferenceResponse struct { + DifferenceStreams map[string]interface{} `json:"difference_streams"` +} + // Job response type JobResponse struct { ID int `json:"id"` @@ -54,6 +64,7 @@ type JobResponse struct { Frequency string `json:"frequency"` LastRunTime string `json:"last_run_time,omitempty"` LastRunState string `json:"last_run_state,omitempty"` + LastRunType string `json:"last_run_type,omitempty"` // "sync" | "clear-destination" CreatedAt string `json:"created_at"` UpdatedAt string `json:"updated_at"` Activate bool `json:"activate"` @@ -66,6 +77,7 @@ type JobTask struct { StartTime string `json:"start_time"` Status string `json:"status"` FilePath string `json:"file_path"` + JobType string `json:"job_type"` // "sync" | "clear-destination" } type SourceDataItem struct { diff --git a/server/internal/handlers/job.go b/server/internal/handlers/job.go index fc32225f..c5b374c4 100644 --- a/server/internal/handlers/job.go +++ b/server/internal/handlers/job.go @@ -3,6 +3,7 @@ package handlers import ( "context" "encoding/json" + "errors" "net/http" "github.com/beego/beego/v2/server/web" @@ -70,6 +71,10 @@ func (c *JobHandler) UpdateJob() { userID := GetUserIDFromSession(&c.Controller) if err := c.jobService.UpdateJob(c.Ctx.Request.Context(), &req, projectID, jobID, userID); err != nil { + if errors.Is(err, constants.ErrInProgress) { + respondWithError(&c.Controller, http.StatusConflict, "Clear destination operation is in progress", err) + return + } respondWithError(&c.Controller, http.StatusInternalServerError, "Failed to update job", err) return } @@ -93,6 +98,10 @@ func (c *JobHandler) SyncJob() { id := GetIDFromPath(&c.Controller) result, err := c.jobService.SyncJob(c.Ctx.Request.Context(), projectID, id) if err != nil { + if errors.Is(err, constants.ErrInProgress) { + respondWithError(&c.Controller, http.StatusConflict, "Failed to sync job, clear destination is running", err) + return + } respondWithError(&c.Controller, http.StatusInternalServerError, "Failed to sync job", err) return } @@ -125,7 +134,7 @@ func (c *JobHandler) ActivateJob() { func (c *JobHandler) CancelJobRun() { projectID := c.Ctx.Input.Param(":projectid") id := GetIDFromPath(&c.Controller) - resp, err := c.jobService.CancelJobRun(c.Ctx.Request.Context(), projectID, id) + resp, err := c.jobService.CancelJobRun(projectID, id) if err != nil { respondWithError(&c.Controller, http.StatusInternalServerError, "Failed to cancel job run", err) return @@ -186,7 +195,45 @@ func (c *JobHandler) CheckUniqueJobName() { return } utils.SuccessResponse(&c.Controller, dto.CheckUniqueJobNameResponse{Unique: unique}) +} + +// @router /project/:projectid/jobs/:id/clear-destination [post] +func (c *JobHandler) ClearDestination() { + projectID := c.Ctx.Input.Param(":projectid") + id := GetIDFromPath(&c.Controller) + + result, err := c.jobService.ClearDestination(c.Ctx.Request.Context(), projectID, id, "") + if err != nil { + if errors.Is(err, constants.ErrInProgress) { + respondWithError(&c.Controller, http.StatusConflict, "operation already in progress:", err) + return + } + respondWithError(&c.Controller, http.StatusInternalServerError, "failed to clear destination", err) + return + } + utils.SuccessResponse(&c.Controller, result) +} + +// @router /project/:projectid/jobs/:id/difference-streams [post] +func (c *JobHandler) DifferenceStreams() { + projectID := c.Ctx.Input.Param(":projectid") + jobID := GetIDFromPath(&c.Controller) + + var req dto.StreamDifferenceRequest + if err := UnmarshalAndValidate(c.Ctx.Input.RequestBody, &req); err != nil { + respondWithError(&c.Controller, http.StatusInternalServerError, constants.ValidationInvalidRequestFormat, err) + return + } + + diffStreams, err := c.jobService.DifferenceStreams(c.Ctx.Request.Context(), projectID, jobID, req) + if err != nil { + respondWithError(&c.Controller, http.StatusInternalServerError, "stream difference command failed", err) + return + } + utils.SuccessResponse(&c.Controller, dto.StreamDifferenceResponse{ + DifferenceStreams: diffStreams, + }) } // worker handler diff --git a/server/internal/services/destination.go b/server/internal/services/destination.go index e2ed462d..58de87f0 100644 --- a/server/internal/services/destination.go +++ b/server/internal/services/destination.go @@ -133,7 +133,7 @@ func (s *DestinationService) UpdateDestination(ctx context.Context, id int, proj } for _, job := range jobs { - if err := cancelJobWorkflow(s.tempClient, job, projectID); err != nil { + if err := cancelJobWorkflow(s.tempClient, job.ProjectID, job.ID); err != nil { return fmt.Errorf("failed to cancel workflow for job %d: %s", job.ID, err) } } diff --git a/server/internal/services/job.go b/server/internal/services/job.go index cd26de89..6cb22275 100644 --- a/server/internal/services/job.go +++ b/server/internal/services/job.go @@ -120,6 +120,42 @@ func (s *JobService) UpdateJob(ctx context.Context, req *dto.UpdateJobRequest, p return fmt.Errorf("job not found: %s", err) } + // block when clear-destination is running + clearRunning, err := isClearRunning(ctx, s.tempClient, projectID, jobID) + if err != nil { + return fmt.Errorf("failed to get clear destination status: %s", err) + } + if clearRunning { + return fmt.Errorf("clear-destination is in progress: %w", constants.ErrInProgress) + } + + // cancel sync before updating the job + syncRunning, err := isSyncRunning(ctx, s.tempClient, projectID, jobID) + if err != nil { + return fmt.Errorf("failed to get sync status: %w", err) + } + if syncRunning { + logs.Info("sync is running for job %d, initiating cancel sync workflow", jobID) + if err := cancelJobWorkflow(s.tempClient, projectID, jobID); err != nil { + return err + } + logs.Info("successfully cancelled sync for job %d", jobID) + } + + // start clear destination + var diffCatalog map[string]interface{} + if err := json.Unmarshal([]byte(req.DifferenceStreams), &diffCatalog); err != nil { + return fmt.Errorf("invalid difference_streams JSON: %s", err) + } + + if len(diffCatalog) > 0 { + logs.Info("stream difference detected for job %d, running clear destination workflow", existingJob.ID) + if _, err := s.ClearDestination(ctx, projectID, jobID, req.DifferenceStreams); err != nil { + return fmt.Errorf("failed to run clear destination workflow: %s", err) + } + logs.Info("successfully triggered clear destination workflow for job %d", existingJob.ID) + } + source, err := s.getOrCreateSource(req.Source, projectID, userID) if err != nil { return fmt.Errorf("failed to process source: %s", err) @@ -193,6 +229,18 @@ func (s *JobService) SyncJob(ctx context.Context, projectID string, jobID int) ( return nil, fmt.Errorf("job must have both source and destination configured") } + if !job.Active { + return nil, fmt.Errorf("job is paused, please unpause to sync job") + } + + running, err := isClearRunning(ctx, s.tempClient, projectID, jobID) + if err != nil { + return nil, fmt.Errorf("failed to get clear destination status: %s", err) + } + if running { + return nil, fmt.Errorf("failed to sync job, clear destination is running: %w", constants.ErrInProgress) + } + if s.tempClient != nil { resp, err := s.tempClient.ManageSync(ctx, job, temporal.ActionTrigger) if err != nil { @@ -204,12 +252,12 @@ func (s *JobService) SyncJob(ctx context.Context, projectID string, jobID int) ( return nil, fmt.Errorf("temporal client is not available") } -func (s *JobService) CancelJobRun(ctx context.Context, projectID string, jobID int) (map[string]any, error) { - job, err := s.jobORM.GetByID(jobID, true) - if err != nil { +func (s *JobService) CancelJobRun(projectID string, jobID int) (map[string]any, error) { + if _, err := s.jobORM.GetByID(jobID, true); err != nil { return nil, fmt.Errorf("job not found id %d: %s", jobID, err) } - if err := cancelJobWorkflow(s.tempClient, job, projectID); err != nil { + + if err := cancelJobWorkflow(s.tempClient, projectID, jobID); err != nil { return nil, fmt.Errorf("job workflow cancel failed id %d: %s", jobID, err) } return map[string]any{ @@ -242,14 +290,27 @@ func (s *JobService) ActivateJob(ctx context.Context, jobID int, activate bool, return nil } +func (s *JobService) DifferenceStreams(ctx context.Context, projectID string, jobID int, req dto.StreamDifferenceRequest) (map[string]interface{}, error) { + existingJob, err := s.jobORM.GetByID(jobID, true) + if err != nil { + return nil, fmt.Errorf("job not found id %d: %s", jobID, err) + } + + diffCatalog, err := s.tempClient.GetDifferenceStreams(ctx, existingJob, existingJob.StreamsConfig, req.UpdatedStreamsConfig) + if err != nil { + return nil, fmt.Errorf("failed to get stream difference: %s", err) + } + + return diffCatalog, nil +} + func (s *JobService) IsJobNameUnique(ctx context.Context, projectID string, req dto.CheckUniqueJobNameRequest) (bool, error) { logs.Info("Checking if job name is unique: %s", req.JobName) return s.jobORM.IsJobNameUnique(projectID, req.JobName) } func (s *JobService) GetJobTasks(ctx context.Context, projectID string, jobID int) ([]dto.JobTask, error) { - job, err := s.jobORM.GetByID(jobID, true) - if err != nil { + if _, err := s.jobORM.GetByID(jobID, true); err != nil { return nil, fmt.Errorf("job not found id %d: %s", jobID, err) } @@ -258,7 +319,10 @@ func (s *JobService) GetJobTasks(ctx context.Context, projectID string, jobID in } var tasks []dto.JobTask - query := fmt.Sprintf("WorkflowId between 'sync-%s-%d' and 'sync-%s-%d-~'", projectID, job.ID, projectID, job.ID) + syncQuery := fmt.Sprintf("WorkflowId between 'sync-%s-%d' and 'sync-%s-%d-~'", projectID, jobID, projectID, jobID) + clearQuery := fmt.Sprintf("WorkflowId between 'clear-destination-%s-%d' and 'clear-destination-%s-%d-~'", projectID, jobID, projectID, jobID) + + query := fmt.Sprintf("(%s) OR (%s)", syncQuery, clearQuery) resp, err := s.tempClient.ListWorkflow(ctx, &workflowservice.ListWorkflowExecutionsRequest{ Query: query, @@ -275,11 +339,19 @@ func (s *JobService) GetJobTasks(ctx context.Context, projectID string, jobID in } else { runTime = time.Since(startTime).Round(time.Second).String() } + + workflowID := execution.Execution.WorkflowId + jobType := "sync" + if strings.HasPrefix(workflowID, "clear-destination-") { + jobType = "clear" + } + tasks = append(tasks, dto.JobTask{ Runtime: runTime, StartTime: startTime.Format(time.RFC3339), Status: execution.Status.String(), - FilePath: execution.Execution.WorkflowId, + FilePath: workflowID, + JobType: jobType, }) } @@ -288,13 +360,13 @@ func (s *JobService) GetJobTasks(ctx context.Context, projectID string, jobID in func (s *JobService) GetTaskLogs(ctx context.Context, jobID int, filePath string) ([]map[string]interface{}, error) { logs.Info("Getting task logs for job with id: %d", jobID) - _, err := s.jobORM.GetByID(jobID, true) - if err != nil { + if _, err := s.jobORM.GetByID(jobID, true); err != nil { return nil, fmt.Errorf("job not found id %d: %s", jobID, err) } syncFolderName := fmt.Sprintf("%x", sha256.Sum256([]byte(filePath))) mainSyncDir := filepath.Join(constants.DefaultConfigDir, syncFolderName) + if _, err := os.Stat(mainSyncDir); os.IsNotExist(err) { return nil, fmt.Errorf("no sync directory found: %s", mainSyncDir) } @@ -378,16 +450,30 @@ func (s *JobService) buildJobResponse(job *models.Job, projectID string) dto.Job } if s.tempClient != nil { - query := fmt.Sprintf("WorkflowId between 'sync-%s-%d' and 'sync-%s-%d-~'", projectID, job.ID, projectID, job.ID) - if resp, err := s.tempClient.ListWorkflow(context.Background(), &workflowservice.ListWorkflowExecutionsRequest{ + syncQuery := fmt.Sprintf("WorkflowId between 'sync-%s-%d' and 'sync-%s-%d-~'", projectID, job.ID, projectID, job.ID) + clearQuery := fmt.Sprintf("WorkflowId between 'clear-destination-%s-%d' and 'clear-destination-%s-%d-~'", projectID, job.ID, projectID, job.ID) + query := fmt.Sprintf("(%s) OR (%s)", syncQuery, clearQuery) + + resp, err := s.tempClient.ListWorkflow(context.Background(), &workflowservice.ListWorkflowExecutionsRequest{ Query: query, PageSize: 1, - }); err == nil && len(resp.Executions) > 0 { - jobResp.LastRunTime = resp.Executions[0].StartTime.AsTime().Format(time.RFC3339) - jobResp.LastRunState = resp.Executions[0].Status.String() + }) + if err != nil { + logs.Error("Failed to list workflows: %s", err) } - } + if len(resp.Executions) > 0 { + execution := resp.Executions[0] + workflowID := execution.Execution.WorkflowId + runType := "sync" + if strings.HasPrefix(workflowID, "clear-destination-") { + runType = "clear" + } + jobResp.LastRunTime = execution.StartTime.AsTime().Format(time.RFC3339) + jobResp.LastRunState = execution.Status.String() + jobResp.LastRunType = runType + } + } return jobResp } @@ -456,6 +542,34 @@ func (s *JobService) getOrCreateDestination(config dto.JobDestinationConfig, pro return dest, nil } +func (s *JobService) ClearDestination(ctx context.Context, projectID string, jobID int, streamsConfig string) (map[string]interface{}, error) { + job, err := s.jobORM.GetByID(jobID, true) + if err != nil { + return nil, fmt.Errorf("job not found id %d: %s", jobID, err) + } + + // block operation if job is paused + if !job.Active { + return nil, fmt.Errorf("job is paused, please unpause to run clear destination") + } + + if running, _ := isClearRunning(ctx, s.tempClient, projectID, jobID); running { + return nil, fmt.Errorf("clear-destination is in progress: %w", constants.ErrInProgress) + } + + // cancel sync if running + if running, _ := isSyncRunning(ctx, s.tempClient, projectID, jobID); running { + return nil, fmt.Errorf("sync is in progress, please cancel it before running clear-destination: %w", constants.ErrInProgress) + } + + result, err := s.tempClient.ClearDestination(ctx, job, streamsConfig) + if err != nil { + return nil, err + } + + return result, nil +} + func (s *JobService) CheckUniqueJobName(projectID string, jobName string) (bool, error) { return s.jobORM.IsJobNameUnique(projectID, jobName) } diff --git a/server/internal/services/source.go b/server/internal/services/source.go index 80af1efd..ea74ba3a 100644 --- a/server/internal/services/source.go +++ b/server/internal/services/source.go @@ -139,7 +139,7 @@ func (s *SourceService) UpdateSource(ctx context.Context, projectID string, id i return fmt.Errorf("failed to fetch jobs for source %d: %s", existing.ID, err) } for _, job := range jobs { - if err := cancelJobWorkflow(s.tempClient, job, projectID); err != nil { + if err := cancelJobWorkflow(s.tempClient, projectID, id); err != nil { return fmt.Errorf("failed to cancel workflow for job %d: %s", job.ID, err) } } diff --git a/server/internal/services/utils.go b/server/internal/services/utils.go index eb218171..c03fa056 100644 --- a/server/internal/services/utils.go +++ b/server/internal/services/utils.go @@ -4,15 +4,14 @@ import ( "context" "fmt" - "github.com/datazip/olake-ui/server/internal/models" "github.com/datazip/olake-ui/server/internal/temporal" "go.temporal.io/api/workflowservice/v1" ) -func cancelJobWorkflow(tempClient *temporal.Client, job *models.Job, projectID string) error { +func cancelJobWorkflow(tempClient *temporal.Client, projectID string, jobID int) error { query := fmt.Sprintf( "WorkflowId BETWEEN 'sync-%s-%d' AND 'sync-%s-%d-~' AND ExecutionStatus = 'Running'", - projectID, job.ID, projectID, job.ID, + projectID, jobID, projectID, jobID, ) resp, err := tempClient.ListWorkflow(context.Background(), &workflowservice.ListWorkflowExecutionsRequest{ @@ -33,3 +32,32 @@ func cancelJobWorkflow(tempClient *temporal.Client, job *models.Job, projectID s } return nil } + +func isClearRunning(ctx context.Context, tempClient *temporal.Client, projectID string, jobID int) (bool, error) { + query := fmt.Sprintf("WorkflowId BETWEEN 'clear-destination-%s-%d' AND 'clear-destination-%s-%d-~' AND ExecutionStatus = 'Running'", + projectID, jobID, projectID, jobID, + ) + + resp, err := tempClient.ListWorkflow(ctx, &workflowservice.ListWorkflowExecutionsRequest{ + Query: query, + }) + if err != nil { + return false, err + } + return len(resp.Executions) > 0, nil +} + +func isSyncRunning(ctx context.Context, tempClient *temporal.Client, projectID string, jobID int) (bool, error) { + query := fmt.Sprintf( + "WorkflowId BETWEEN 'sync-%s-%d' AND 'sync-%s-%d-~' AND ExecutionStatus = 'Running'", + projectID, jobID, projectID, jobID, + ) + + resp, err := tempClient.ListWorkflow(ctx, &workflowservice.ListWorkflowExecutionsRequest{ + Query: query, + }) + if err != nil { + return false, err + } + return len(resp.Executions) > 0, nil +} diff --git a/server/internal/temporal/client.go b/server/internal/temporal/client.go index 65089f22..0ada4e62 100644 --- a/server/internal/temporal/client.go +++ b/server/internal/temporal/client.go @@ -37,6 +37,7 @@ type ExecutionRequest struct { Configs []JobConfig `json:"configs"` WorkflowID string `json:"workflow_id"` JobID int `json:"job_id"` + ProjectID string `json:"project_id"` Timeout time.Duration `json:"timeout"` OutputFile string `json:"output_file"` // to get the output file from the workflow } @@ -57,10 +58,11 @@ const ( ActionPause SyncAction = "pause" ActionUnpause SyncAction = "unpause" - Discover Command = "discover" - Check Command = "check" - Sync Command = "sync" - Spec Command = "spec" + Discover Command = "discover" + Check Command = "check" + Sync Command = "sync" + Spec Command = "spec" + ClearDestination Command = "clear-destination" ) func init() { @@ -262,6 +264,114 @@ func (c *Client) FetchSpec(ctx context.Context, destinationType, sourceType, ver }, nil } +func (c *Client) ClearDestination(ctx context.Context, job *models.Job, streamsConfig string) (map[string]interface{}, error) { + workflowID := fmt.Sprintf("clear-destination-%s-%d-%d", job.ProjectID, job.ID, time.Now().Unix()) + + catalog := utils.Ternary(streamsConfig != "", streamsConfig, job.StreamsConfig).(string) + + // pause the sync schedule + // worker will unpause in cleanup + if _, err := c.ManageSync(ctx, job, ActionPause); err != nil { + return nil, err + } + + configs := []JobConfig{ + {Name: "streams.json", Data: catalog}, + {Name: "state.json", Data: job.State}, + {Name: "destination.json", Data: job.DestID.Config}, + {Name: "user_id.txt", Data: telemetry.GetTelemetryUserID()}, + } + + cmdArgs := []string{ + "clear-destination", + "--streams", "/mnt/config/streams.json", + "--state", "/mnt/config/state.json", + "--destination", "/mnt/config/destination.json", + } + if encryptionKey, _ := web.AppConfig.String("encryptionkey"); encryptionKey != "" { + cmdArgs = append(cmdArgs, "--encryption-key", encryptionKey) + } + + req := &ExecutionRequest{ + Type: "docker", + Command: ClearDestination, + ConnectorType: job.SourceID.Type, + Version: job.SourceID.Version, + Args: cmdArgs, + Configs: configs, + WorkflowID: workflowID, + JobID: job.ID, + ProjectID: job.ProjectID, + Timeout: GetWorkflowTimeout(ClearDestination), + OutputFile: "state.json", + } + + workflowOptions := client.StartWorkflowOptions{ + ID: workflowID, + TaskQueue: TaskQueue, + } + + _, err := c.temporalClient.ExecuteWorkflow(ctx, workflowOptions, "ExecuteClearWorkflow", req) + if err != nil { + _, _ = c.ManageSync(ctx, job, ActionUnpause) + return nil, fmt.Errorf("failed to execute clear destination workflow: %v", err) + } + + return map[string]interface{}{ + "message": "Clear destination initiated successfully", + }, nil +} + +func (c *Client) GetDifferenceStreams(ctx context.Context, job *models.Job, oldConfig, newConfig string) (map[string]interface{}, error) { + workflowID := fmt.Sprintf("difference-%s-%d-%d", job.ProjectID, job.ID, time.Now().Unix()) + + configs := []JobConfig{ + {Name: "old_streams.json", Data: oldConfig}, + {Name: "new_streams.json", Data: newConfig}, + {Name: "user_id.txt", Data: telemetry.GetTelemetryUserID()}, + } + + cmdArgs := []string{ + "discover", + "--streams", "/mnt/config/old_streams.json", + "--difference", "/mnt/config/new_streams.json", + } + if encryptionKey, _ := web.AppConfig.String("encryptionkey"); encryptionKey != "" { + cmdArgs = append(cmdArgs, "--encryption-key", encryptionKey) + } + + req := &ExecutionRequest{ + Type: "docker", + Command: Discover, + ConnectorType: job.SourceID.Type, + Version: job.SourceID.Version, + Args: cmdArgs, + Configs: configs, + WorkflowID: workflowID, + JobID: job.ID, + ProjectID: job.ProjectID, + Timeout: GetWorkflowTimeout(Discover), + OutputFile: "difference_streams.json", + } + + workflowOptions := client.StartWorkflowOptions{ + ID: workflowID, + TaskQueue: TaskQueue, + } + + run, err := c.temporalClient.ExecuteWorkflow(ctx, workflowOptions, "ExecuteWorkflow", req) + if err != nil { + return nil, fmt.Errorf("failed to execute get stream difference workflow: %v", err) + } + + diffStreams, err := ExtractWorkflowResponse(ctx, run) + if err != nil { + return nil, fmt.Errorf("failed to extract workflow response: %v", err) + } + + return diffStreams, nil +} + func (c *Client) ManageSync(ctx context.Context, job *models.Job, action SyncAction) (map[string]interface{}, error) { workflowID := fmt.Sprintf("sync-%s-%d", job.ProjectID, job.ID) scheduleID := fmt.Sprintf("schedule-%s", workflowID) @@ -334,7 +444,7 @@ func (c *Client) createSchedule(ctx context.Context, job *models.Job, scheduleID }, Action: &client.ScheduleWorkflowAction{ ID: workflowID, - Workflow: "ExecuteSyncWorkflow", + Workflow: "RunSyncWorkflow", Args: []any{req}, TaskQueue: TaskQueue, }, @@ -365,7 +475,7 @@ func (c *Client) updateSchedule(ctx context.Context, handle client.ScheduleHandl input.Description.Schedule.Action = &client.ScheduleWorkflowAction{ ID: workflowID, - Workflow: "ExecuteSyncWorkflow", + Workflow: "RunSyncWorkflow", Args: []any{req}, TaskQueue: TaskQueue, } diff --git a/server/internal/temporal/utils.go b/server/internal/temporal/utils.go index 704bca33..d3362ee6 100644 --- a/server/internal/temporal/utils.go +++ b/server/internal/temporal/utils.go @@ -69,6 +69,8 @@ func GetWorkflowTimeout(op Command) time.Duration { return time.Minute * 5 case Sync: return time.Hour * 24 * 30 + case ClearDestination: + return time.Minute * 60 // check what can the fallback time be default: return time.Minute * 5 diff --git a/server/routes/router.go b/server/routes/router.go index 1cdef621..28f9fb8b 100644 --- a/server/routes/router.go +++ b/server/routes/router.go @@ -87,6 +87,8 @@ func Init() { web.Router("/api/v1/project/:projectid/jobs/:id/cancel", &handlers.JobHandler{}, "get:CancelJobRun") web.Router("/api/v1/project/:projectid/jobs/:id/tasks/:taskid/logs", &handlers.JobHandler{}, "post:GetTaskLogs") web.Router("/api/v1/project/:projectid/jobs/check-unique", &handlers.JobHandler{}, "post:CheckUniqueJobName") + web.Router("/api/v1/project/:projectid/jobs/:id/clear-destination", &handlers.JobHandler{}, "post:ClearDestination") + web.Router("/api/v1/project/:projectid/jobs/:id/stream-difference", &handlers.JobHandler{}, "post:DifferenceStreams") // worker callback routes web.Router("/internal/worker/callback/sync-telemetry", &handlers.JobHandler{}, "post:UpdateSyncTelemetry") diff --git a/server/tests/docker-compose.yml b/server/tests/docker-compose.yml new file mode 100644 index 00000000..f07ec699 --- /dev/null +++ b/server/tests/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3.8' + +services: + postgres: + image: postgres:15 + container_name: olake_postgres-test + restart: unless-stopped + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret1234 + POSTGRES_DB: postgres + ports: + - "5433:5432" + volumes: + - ./pg_hba.conf:/etc/postgresql/pg_hba.conf + command: + - bash + - -c + - | + # Install wal2json + apt-get update && apt-get install -y postgresql-15-wal2json + + # Start PostgreSQL with basic config in background + docker-entrypoint.sh postgres \ + -c hba_file=/etc/postgresql/pg_hba.conf \ + -c listen_addresses='*' \ + -c wal_level=logical \ + -c max_wal_senders=10 \ + -c max_replication_slots=10 \ + -c shared_preload_libraries=wal2json & + + # Wait for PostgreSQL to be ready + while ! pg_isready -U postgres -h localhost -p 5432; do + sleep 1 + done + + # Create replication slot if it doesn't exist + psql -v ON_ERROR_STOP=1 -U postgres < /dev/null && + apt-get update && + apt-get install -y docker-compose-plugin && + update-ca-certificates + ` + + // Download destination docker-compose + // TODO: Either move destination and source config into the same Docker Compose setup or download both from the same location for consistency. + downloadDestinationComposeCmd = ` + cd /mnt && + curl -fsSL -o docker-compose.destination.yml \ + https://raw.githubusercontent.com/datazip-inc/olake/master/destination/iceberg/local-test/docker-compose.yml + ` + + // Start postgres test infrastructure + startPostgresCmd = ` + cd /mnt/server/tests && + docker compose up -d && + for i in $(seq 1 30); do + if docker exec olake_postgres-test psql -h localhost -U postgres -d postgres -c "SELECT 1" 2>/dev/null; then + echo "PostgreSQL ready." + break + fi + sleep 2 + done && + docker exec olake_postgres-test psql -U postgres -d postgres -c \ + "SELECT slot_name, plugin, slot_type, active FROM pg_replication_slots WHERE slot_name = 'olake_slot';" + ` + + // Start destination services (iceberg stack) + startDestinationCmd = ` + cd /mnt && + docker compose -f docker-compose.destination.yml up -d minio mc postgres spark-iceberg && + sleep 5 && + docker compose -f docker-compose.destination.yml ps + ` + + // Start OLake application + startOLakeCmd = ` + cd /mnt && + mkdir -p /mnt/olake-data && + docker compose up -d && + for i in $(seq 1 60); do + if curl -f http://localhost:8000/health 2>/dev/null || curl -f http://localhost:8000 2>/dev/null; then + echo "OLake UI ready." + break + fi + sleep 2 + done + ` + + // Network setup + networkSetupCmd = ` + docker network create olake-network || true && + docker network connect olake-network olake-ui || true && + docker network connect olake-network postgres || true && + docker network connect olake-network olake_postgres-test || true + ` + + // Install Playwright and dependencies + installPlaywrightCmd = ` + cd /mnt/ui && + pnpm add -D @playwright/test && + pnpm exec playwright install --with-deps chromium + ` + + // Run Playwright tests + runPlaywrightCmd = ` + cd /mnt/ui && + PLAYWRIGHT_TEST_BASE_URL=http://localhost:8000 DEBUG=pw:api npx playwright test tests/flows/job-end-to-end.spec.ts + ` + + icebergDB = "postgres_iceberg_jdbc_job_postgres_public" + icebergCatalog = "olake_iceberg" + currentTestTable = "postgres_test_table_olake" +) + +func DinDTestContainer(t *testing.T) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + projectRoot, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + return fmt.Errorf("could not determine project root: %w", err) + } + t.Logf("Project root identified at: %s", projectRoot) + + req := testcontainers.ContainerRequest{ + Image: "ubuntu:22.04", + Env: map[string]string{ + "DOCKER_TLS_CERTDIR": "", + "TELEMETRY_DISABLED": "true", + "TESTCONTAINERS_RYUK_DISABLED": "true", + "DEBIAN_FRONTEND": "noninteractive", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.Privileged = true + hc.Binds = []string{ + fmt.Sprintf("%s:/mnt:rw", projectRoot), + } + // Tmpfs mounts create temporary in-memory filesystems inside the container. + // These directories behave like RAM disks they exist only in memory (not on disk) and are automatically cleaned up when the container stops. + // This is useful for high-performance temporary storage. + // 70GB for docker and 15GB for shared memory space in Linux + hc.Tmpfs = map[string]string{ + "/var/lib/docker": "size=70G", + "/dev/shm": "size=10G", + } + hc.Resources.Memory = 22 * 1024 * 1024 * 1024 // 22GB + hc.ExtraHosts = append(hc.ExtraHosts, "host.docker.internal:host-gateway") + }, + ConfigModifier: func(config *container.Config) { + config.WorkingDir = "/mnt" + }, + ExposedPorts: []string{"8000/tcp", "2375/tcp", "5433/tcp", "15002/tcp"}, + Cmd: []string{ + "/bin/sh", "-c", + `set -e + apt-get update -y && + apt-get install -y --no-install-recommends ca-certificates curl gnupg lsb-release iproute2 procps && + apt-get install -y --no-install-recommends docker.io && + mkdir -p /var/lib/docker /var/run/docker && + exec dockerd --host=tcp://0.0.0.0:2375 --host=unix:///var/run/docker.sock + `, + }, + WaitingFor: wait.ForExec([]string{"docker", "-H", "tcp://127.0.0.1:2375", "info"}).WithStartupTimeout(60 * time.Second).WithPollInterval(1 * time.Second), + } + + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return fmt.Errorf("failed to start DinD container: %w", err) + } + + // container host + host, err := ctr.Host(ctx) + if err != nil { + return fmt.Errorf("failed to get host: %w", err) + } + + // postgres port + postgresPort, err := ctr.MappedPort(ctx, "5433/tcp") + if err != nil { + return fmt.Errorf("failed to get postgres port: %w", err) + } + + // iceberg-spark port + sparkPort, err := ctr.MappedPort(ctx, "15002/tcp") + if err != nil { + return fmt.Errorf("failed to get spark port: %w", err) + } + + t.Log("Docker daemon is ready") + // Step 1: Install tools + t.Log("Installing required tools...") + if code, out, err := ExecCommand(ctx, ctr, setupToolsCmd); err != nil || code != 0 { + return fmt.Errorf("tools installation failed (%d): %s\n%s", code, err, out) + } + + // Step 2: Download destination docker-compose + t.Log("Downloading destination docker-compose...") + if code, out, err := ExecCommand(ctx, ctr, downloadDestinationComposeCmd); err != nil || code != 0 { + return fmt.Errorf("destination docker-compose download failed (%d): %s\n%s", code, err, out) + } + + // Step 3: Start PostgreSQL test infrastructure + t.Log("Starting PostgreSQL test infrastructure...") + if code, out, err := ExecCommand(ctx, ctr, startPostgresCmd); err != nil || code != 0 { + return fmt.Errorf("postgres startup failed (%d): %s\n%s", code, err, out) + } + + // Step 4: Start destination services (Iceberg stack) + t.Log("Starting destination services...") + if code, out, err := ExecCommand(ctx, ctr, startDestinationCmd); err != nil || code != 0 { + return fmt.Errorf("destination services startup failed (%d): %s\n%s", code, err, out) + } + + // Step 5: Patch docker-compose for local images + t.Log("Patching docker-compose to build local images...") + if err := PatchDockerCompose(ctx, t, ctr); err != nil { + return err + } + + // Step 6: Start OLake application + t.Log("Starting OLake docker-compose services...") + if code, out, err := ExecCommand(ctx, ctr, startOLakeCmd); err != nil || code != 0 { + return fmt.Errorf("OLake startup failed (%d): %s\n%s", code, err, out) + } + + // Step 7: Setup networks + t.Log("Setting up Docker networks...") + if code, out, err := ExecCommand(ctx, ctr, networkSetupCmd); err != nil || code != 0 { + t.Logf("Warning: Network setup failed (%d): %s\n%s", code, err, out) + } + + // Step 8: Query the postgres source + ExecuteQuery(ctx, t, "create", host, postgresPort.Port()) + ExecuteQuery(ctx, t, "clean", host, postgresPort.Port()) + ExecuteQuery(ctx, t, "add", host, postgresPort.Port()) + + t.Logf("OLake UI is ready and accessible at: http://localhost:8000") + + // Step 9: Install Playwright + t.Log("Installing Playwright and dependencies...") + if code, out, err := ExecCommand(ctx, ctr, installPlaywrightCmd); err != nil || code != 0 { + return fmt.Errorf("playwright installation failed (%d): %s\n%s", code, err, out) + } + + // Step 10: Run Playwright tests + t.Log("Executing Playwright tests...") + if code, out, err := ExecCommandWithStreaming(ctx, t, ctr, runPlaywrightCmd); err != nil || code != 0 { + return fmt.Errorf("playwright tests failed (%d): %s\n%s", code, err, out) + } + t.Log("Playwright tests passed successfully.") + + // Step 11: Verify in iceberg + t.Logf("Starting Iceberg data verification...") + VerifyIcebergTest(ctx, t, ctr, host, sparkPort.Port()) + return nil +} + +// ExecCommandWithStreaming executes a command and streams output in real-time +func ExecCommandWithStreaming(ctx context.Context, t *testing.T, ctr testcontainers.Container, cmd string) (int, string, error) { + exitCode, reader, err := ctr.Exec(ctx, []string{"sh", "-c", cmd}) + if err != nil { + return -1, "", err + } + + var output strings.Builder + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := scanner.Text() + t.Log(line) + output.WriteString(line + "\n") + } + + if err := scanner.Err(); err != nil { + return exitCode, output.String(), err + } + + return exitCode, output.String(), nil +} + +// PatchDockerCompose updates olake-ui and temporal-worker to build from local code +// TODO: Remove patch command and find alternative to use local code +func PatchDockerCompose(ctx context.Context, t *testing.T, ctr testcontainers.Container) error { + patchCmd := ` + set -e + tmpfile=$(mktemp) + awk ' + BEGIN{svc="";} + /^ olake-ui:/{svc="olake-ui"; print; next} + /^ temporal-worker:/{svc="temporal-worker"; print; next} + /^ [A-Za-z0-9_-]+:/{ if (svc!="") svc=""; print; next} + { + if (svc=="olake-ui" && $0 ~ /^ image:/) { + print " build:"; + print " context: ."; + print " dockerfile: Dockerfile"; + next + } + if (svc=="temporal-worker" && $0 ~ /^ image:/) { + print " build:"; + print " context: ."; + print " dockerfile: worker.Dockerfile"; + next + } + print + } + ' /mnt/docker-compose.yml > "$tmpfile" && mv "$tmpfile" /mnt/docker-compose.yml +` + + code, out, err := ExecCommand(ctx, ctr, patchCmd) + if err != nil || code != 0 { + t.Logf("docker-compose patch output: %s", string(out)) + return fmt.Errorf("failed to patch docker-compose.yml (%d): %s\n%s", code, err, out) + } + t.Log("docker-compose.yml patched to build local images") + t.Logf("Patched docker-compose.yml:\n%s", string(out)) + + return nil +} + +func VerifyIcebergTest(ctx context.Context, t *testing.T, ctr testcontainers.Container, host, port string) { + sparkConnectAddress := fmt.Sprintf("sc://%s:%s", host, port) + spark, err := sql.NewSessionBuilder().Remote(sparkConnectAddress).Build(ctx) + require.NoError(t, err, "Failed to connect to Spark Connect server") + defer func() { + if stopErr := spark.Stop(); stopErr != nil { + t.Errorf("Failed to stop Spark session: %v", stopErr) + } + if ctr != nil { + t.Log("Running cleanup...") + // Stop docker-compose services + _, _, _ = ExecCommand(ctx, ctr, "cd /mnt && docker-compose down -v --remove-orphans") + // Terminate the DinD container + if err := ctr.Terminate(ctx); err != nil { + t.Logf("Warning: failed to terminate container: %v", err) + } + t.Log("Cleanup complete") + } + }() + countQuery := fmt.Sprintf( + "SELECT COUNT(DISTINCT _olake_id) as unique_count FROM %s.%s.%s", + icebergCatalog, icebergDB, currentTestTable, + ) + t.Logf("Executing query: %s", countQuery) + + countQueryDf, err := spark.Sql(ctx, countQuery) + require.NoError(t, err, "Failed to execute query on the table") + + rows, err := countQueryDf.Collect(ctx) + require.NoError(t, err, "Failed to collect data rows from Iceberg") + require.NotEmpty(t, rows, "No rows returned for _op_type = 'r'") + + // check count and verify + countValue := rows[0].Value("unique_count").(int64) + require.Equal(t, int64(5), countValue, "Expected count to be 5") + t.Logf("✅ Test passed: count value %v matches expected value 5", countValue) +} + +func ExecuteQuery(ctx context.Context, t *testing.T, operation, host, port string) { + t.Helper() + connStr := fmt.Sprintf("postgres://postgres@%s:%s/postgres?sslmode=disable", host, port) + db, ok := sqlx.ConnectContext(ctx, "postgres", connStr) + require.NoError(t, ok, "failed to connect to postgres") + defer func() { + require.NoError(t, db.Close(), "failed to close postgres connection") + }() + + // integration test uses only one stream for testing + integrationTestTable := currentTestTable + var query string + + switch operation { + case "create": + query = fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + col_bigint BIGINT, + col_bigserial BIGSERIAL PRIMARY KEY, + col_bool BOOLEAN, + col_char CHAR(1), + col_character CHAR(10), + col_character_varying VARCHAR(50), + col_date DATE, + col_decimal NUMERIC, + col_double_precision DOUBLE PRECISION, + col_float4 REAL, + col_int INT, + col_int2 SMALLINT, + col_integer INTEGER, + col_interval INTERVAL, + col_json JSON, + col_jsonb JSONB, + col_name NAME, + col_numeric NUMERIC, + col_real REAL, + col_text TEXT, + col_timestamp TIMESTAMP, + col_timestamptz TIMESTAMPTZ, + col_uuid UUID, + col_varbit VARBIT(20), + col_xml XML, + CONSTRAINT unique_custom_key UNIQUE (col_bigserial) + )`, integrationTestTable) + + case "drop": + query = fmt.Sprintf("DROP TABLE IF EXISTS %s", integrationTestTable) + + case "clean": + query = fmt.Sprintf("TRUNCATE TABLE %s", integrationTestTable) + + case "add": + insertTestData(ctx, t, db, integrationTestTable) + return // Early return since we handle all inserts in the helper function + + case "insert": + query = fmt.Sprintf(` + INSERT INTO %s ( + col_bigint, col_bool, col_char, col_character, + col_character_varying, col_date, col_decimal, + col_double_precision, col_float4, col_int, col_int2, + col_integer, col_interval, col_json, col_jsonb, + col_name, col_numeric, col_real, col_text, + col_timestamp, col_timestamptz, col_uuid, col_varbit, col_xml + ) VALUES ( + 123456789012345, TRUE, 'c', 'char_val', + 'varchar_val', '2023-01-01', 123.45, + 123.456789, 123.45, 123, 123, 12345, + '1 hour', '{"key": "value"}', '{"key": "value"}', + 'test_name', 123.45, 123.45, 'sample text', + '2023-01-01 12:00:00', '2023-01-01 12:00:00+00', + '123e4567-e89b-12d3-a456-426614174000', B'101010', + 'value' + )`, integrationTestTable) + + case "update": + query = fmt.Sprintf(` + UPDATE %s SET + col_bigint = 123456789012340, + col_bool = FALSE, + col_char = 'd', + col_character = 'updated__', + col_character_varying = 'updated val', + col_date = '2024-07-01', + col_decimal = 543.21, + col_double_precision = 987.654321, + col_float4 = 543.21, + col_int = 321, + col_int2 = 321, + col_integer = 54321, + col_interval = '2 hours', + col_json = '{"new": "json"}', + col_jsonb = '{"new": "jsonb"}', + col_name = 'updated_name', + col_numeric = 321.00, + col_real = 321.00, + col_text = 'updated text', + col_timestamp = '2024-07-01 15:30:00', + col_timestamptz = '2024-07-01 15:30:00+00', + col_uuid = '00000000-0000-0000-0000-000000000000', + col_varbit = B'111000', + col_xml = 'value' + WHERE col_bigserial = 1`, integrationTestTable) + + case "delete": + query = fmt.Sprintf("DELETE FROM %s WHERE col_bigserial = 1", integrationTestTable) + + default: + t.Fatalf("Unsupported operation: %s", operation) + } + _, err := db.ExecContext(ctx, query) + require.NoError(t, err, "Failed to execute %s operation", operation) +} + +// insertTestData inserts test data into the specified table +func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName string) { + t.Helper() + + for i := 1; i <= 5; i++ { + query := fmt.Sprintf(` + INSERT INTO %s ( + col_bigint, col_bigserial, col_bool, col_char, col_character, + col_character_varying, col_date, col_decimal, + col_double_precision, col_float4, col_int, col_int2, col_integer, + col_interval, col_json, col_jsonb, col_name, col_numeric, + col_real, col_text, col_timestamp, col_timestamptz, + col_uuid, col_varbit, col_xml + ) VALUES ( + 123456789012345, DEFAULT, TRUE, 'c', 'char_val', + 'varchar_val', '2023-01-01', 123.45, + 123.456789, 123.45, 123, 123, 12345, '1 hour', '{"key": "value"}', + '{"key": "value"}', 'test_name', 123.45, 123.45, + 'sample text', '2023-01-01 12:00:00', + '2023-01-01 12:00:00+00', + '123e4567-e89b-12d3-a456-426614174000', B'101010', + 'value' + )`, tableName) + + _, err := db.ExecContext(ctx, query) + require.NoError(t, err, "Failed to insert test data") + } +} + +// Helper function to execute container commands +func ExecCommand( + ctx context.Context, + c testcontainers.Container, + cmd string, +) (int, []byte, error) { + code, reader, err := c.Exec(ctx, []string{"/bin/sh", "-c", cmd}) + if err != nil { + return code, nil, err + } + output, _ := io.ReadAll(reader) + return code, output, nil +} diff --git a/server/tests/verify_test.go b/server/tests/verify_test.go new file mode 100644 index 00000000..49a31912 --- /dev/null +++ b/server/tests/verify_test.go @@ -0,0 +1,16 @@ +package tests + +import ( + "os" + "testing" + + _ "github.com/lib/pq" +) + +func TestDinDIntegration(t *testing.T) { + os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") + err := DinDTestContainer(t) + if err != nil { + t.Errorf("Error in Docker in Docker container start up: %s", err) + } +} diff --git a/server/utils/utils.go b/server/utils/utils.go index b0f39206..661857fb 100644 --- a/server/utils/utils.go +++ b/server/utils/utils.go @@ -199,4 +199,77 @@ func ToCron(frequency string) string { } } +// LogEntry represents a log entry +type LogEntry struct { + Level string `json:"level"` + Time time.Time `json:"time"` + Message json.RawMessage `json:"message"` // store raw JSON +} + +// ReadLogs reads logs from the given mainLogDir and returns structured log entries. +func ReadLogs(mainLogDir string) ([]map[string]interface{}, error) { + // Check if mainLogDir exists + if _, err := os.Stat(mainLogDir); os.IsNotExist(err) { + return nil, fmt.Errorf("logs directory not found: %s", mainLogDir) + } + + // Logs directory + logsDir := filepath.Join(mainLogDir, "logs") + if _, err := os.Stat(logsDir); os.IsNotExist(err) { + return nil, fmt.Errorf("logs directory not found: %s", logsDir) + } + + files, err := os.ReadDir(logsDir) + if err != nil || len(files) == 0 { + return nil, fmt.Errorf("logs directory empty in: %s", logsDir) + } + + logDir := filepath.Join(logsDir, files[0].Name()) + logPath := filepath.Join(logDir, "olake.log") + logContent, err := os.ReadFile(logPath) + if err != nil { + return nil, fmt.Errorf("failed to read log file: %s", logPath) + } + + var parsedLogs []map[string]interface{} + lines := strings.Split(string(logContent), "\n") + for _, line := range lines { + if line == "" { + continue + } + + var logEntry LogEntry + if err := json.Unmarshal([]byte(line), &logEntry); err != nil { + continue + } + + if logEntry.Level == "debug" { + continue + } + + // Convert Message to string safely + var messageStr string + var tmp interface{} + if err := json.Unmarshal(logEntry.Message, &tmp); err == nil { + switch v := tmp.(type) { + case string: + messageStr = v // plain string + default: + msgBytes, _ := json.Marshal(v) // object/array + messageStr = string(msgBytes) + } + } else { + // fallback: raw bytes as string + messageStr = string(logEntry.Message) + } + + parsedLogs = append(parsedLogs, map[string]interface{}{ + "level": logEntry.Level, + "time": logEntry.Time.UTC().Format(time.RFC3339), + "message": messageStr, + }) + } + + return parsedLogs, nil +} diff --git a/ui/package.json b/ui/package.json index 24407fc4..ad7531d1 100644 --- a/ui/package.json +++ b/ui/package.json @@ -14,7 +14,10 @@ "lint:fix": "eslint \"**/*.{js,jsx,ts,tsx}\" --max-warnings 0 --fix", "preview": "vite preview", "format": "prettier --write \"**/*.{js,jsx,ts,tsx}\"", - "format:check": "prettier --check \"**/*.{js,jsx,ts,tsx}\"" + "format:check": "prettier --check \"**/*.{js,jsx,ts,tsx}\"", + "test": "playwright test", + "test:headed": "playwright test --headed", + "test:codegen": "playwright codegen http://localhost:8000/login" }, "dependencies": { "@babel/runtime": "7.26.10", @@ -36,6 +39,7 @@ }, "devDependencies": { "@eslint/js": "^9.19.0", + "@playwright/test": "^1.55.0", "@types/json-schema": "^7.0.15", "@types/node": "^22.13.10", "@types/react": "^18.2.62", diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts new file mode 100644 index 00000000..88062caa --- /dev/null +++ b/ui/playwright.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, devices } from "@playwright/test" + +//TODO: Decide on Timeouts and adjust them as needed + +// Constants +export const TIMEOUTS = { + LONG: 5 * 60 * 1000, // 5 minutes + SHORT: 10 * 1000, // 10 seconds +} as const + +export default defineConfig({ + testDir: "./tests", + timeout: 10 * 60 * 1000, // 10 minutes global test timeout + expect: { + timeout: 5 * 60 * 1000, // 5 minutes for expect assertions + }, + use: { + baseURL: "http://localhost:8000", + screenshot: "only-on-failure", + actionTimeout: 5 * 60 * 1000, // 5 minutes for actions + navigationTimeout: 30 * 1000, // 30 seconds for navigation + }, + projects: [ + // Setup project - runs first to create authenticated state + { + name: "setup", + testMatch: /.*\.setup\.ts/, + }, + // Main test project - depends on setup + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + dependencies: ["setup"], + }, + ], +}) diff --git a/ui/pnpm-lock.yaml b/ui/pnpm-lock.yaml index 7c30b8ee..402d006c 100644 --- a/ui/pnpm-lock.yaml +++ b/ui/pnpm-lock.yaml @@ -64,6 +64,9 @@ importers: '@eslint/js': specifier: ^9.19.0 version: 9.35.0 + '@playwright/test': + specifier: ^1.55.0 + version: 1.55.1 '@types/json-schema': specifier: ^7.0.15 version: 7.0.15 @@ -521,6 +524,11 @@ packages: resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + '@playwright/test@1.55.1': + resolution: {integrity: sha512-IVAh/nOJaw6W9g+RJVlIQJ6gSiER+ae6mKQ5CX1bERzQgbC1VSeBlwdvczT7pxb0GWiyrxH4TGKbMfDb4Sq/ig==} + engines: {node: '>=18'} + hasBin: true + '@rc-component/async-validator@5.0.4': resolution: {integrity: sha512-qgGdcVIF604M9EqjNF0hbUTz42bz/RDtxWdWuU5EQe3hi7M8ob54B6B35rOsvX5eSvIHIzT9iH1R3n+hk3CGfg==} engines: {node: '>=14.x'} @@ -1309,6 +1317,11 @@ packages: fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1785,6 +1798,16 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} + playwright-core@1.55.1: + resolution: {integrity: sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.55.1: + resolution: {integrity: sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A==} + engines: {node: '>=18'} + hasBin: true + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -2952,6 +2975,10 @@ snapshots: '@pkgr/core@0.2.9': {} + '@playwright/test@1.55.1': + dependencies: + playwright: 1.55.1 + '@rc-component/async-validator@5.0.4': dependencies: '@babel/runtime': 7.26.10 @@ -3963,6 +3990,9 @@ snapshots: fraction.js@4.3.7: {} + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -4421,6 +4451,14 @@ snapshots: pirates@4.0.7: {} + playwright-core@1.55.1: {} + + playwright@1.55.1: + dependencies: + playwright-core: 1.55.1 + optionalDependencies: + fsevents: 2.3.2 + possible-typed-array-names@1.1.0: {} postcss-import@15.1.0(postcss@8.5.6): diff --git a/ui/public/vite.svg b/ui/public/vite.svg deleted file mode 100644 index e7b8dfb1..00000000 --- a/ui/public/vite.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/ui/src/api/services/analyticsService.ts b/ui/src/api/services/analyticsService.ts index e3212c94..1c555514 100644 --- a/ui/src/api/services/analyticsService.ts +++ b/ui/src/api/services/analyticsService.ts @@ -1,6 +1,11 @@ +/** + * AnalyticsService handles sending analytics events + */ + import api from "../axios" import axios from "axios" +// endpoint which handles rate limiting and forwards the events to mixpanel const ANALYTICS_ENDPOINT = "https://analytics.olake.io/mp/track" const sendAnalyticsEvent = async ( @@ -57,6 +62,7 @@ const getSystemInfo = async () => { } } +// returns a unique user id for the user to track them across sessions const getTelemetryID = async (): Promise => { try { const response = await api.get("/telemetry-id") @@ -77,6 +83,7 @@ export const trackEvent = async ( return } + // if user is already logged in we'll get the username from local storage const username = localStorage.getItem("username") const systemInfo = await getSystemInfo() diff --git a/ui/src/api/services/destinationService.ts b/ui/src/api/services/destinationService.ts index 09afd4f9..e8345177 100644 --- a/ui/src/api/services/destinationService.ts +++ b/ui/src/api/services/destinationService.ts @@ -11,6 +11,7 @@ import { getConnectorInLowerCase } from "../../utils/utils" // TODO: Make it parquet on all places const normalizeDestinationType = (type: string): string => { + //destination connector typemap const typeMap: Record = { "amazon s3": "s3", "apache iceberg": "iceberg", @@ -93,6 +94,7 @@ export const destinationService = { source_type: source_type, source_version: source_version, }, + //timeout is 0 as test connection takes more time as it needs to connect to the destination { timeout: 0 }, ) return { @@ -136,6 +138,7 @@ export const destinationService = { source_type: source_type, source_version: source_version, }, + //timeout is 300000 as spec takes more time as it needs to fetch the spec from the destination { timeout: 300000, signal }, ) return response.data diff --git a/ui/src/api/services/jobService.ts b/ui/src/api/services/jobService.ts index 7d392727..d916ffee 100644 --- a/ui/src/api/services/jobService.ts +++ b/ui/src/api/services/jobService.ts @@ -1,6 +1,14 @@ import api from "../axios" import { API_CONFIG } from "../config" -import { APIResponse, Job, JobBase, JobTask, TaskLog } from "../../types" +import { + APIResponse, + Job, + JobBase, + JobTask, + StreamsDataStructure, + TaskLog, +} from "../../types" +import { AxiosError } from "axios" export const jobService = { getJobs: async (): Promise => { @@ -75,7 +83,9 @@ export const jobService = { return response.data } catch (error) { console.error("Error syncing job:", error) - throw error + throw error instanceof AxiosError && error.response?.data.message + ? error.response?.data.message + : "Failed to sync job" } }, @@ -101,7 +111,7 @@ export const jobService = { const response = await api.post>( `${API_CONFIG.ENDPOINTS.JOBS(API_CONFIG.PROJECT_ID)}/${jobId}/tasks/${taskId}/logs`, { file_path: filePath }, - { timeout: 0 }, + { timeout: 0 }, // Disable timeout for this request since it can take longer ) return response.data } catch (error) { @@ -110,6 +120,7 @@ export const jobService = { } }, + //This either pauses or resumes the job activateJob: async ( jobId: string, activate: boolean, @@ -138,4 +149,38 @@ export const jobService = { throw error } }, + + clearDestination: async ( + jobId: string, + ): Promise> => { + try { + const response = await api.post>( + `${API_CONFIG.ENDPOINTS.JOBS(API_CONFIG.PROJECT_ID)}/${jobId}/clear-destination`, + ) + + return response.data + } catch (error) { + console.error("Error clearing destination:", error) + throw error instanceof AxiosError && error.response?.data.message + ? `Failed to clear destination: ${error.response?.data.message}` + : "Failed to clear destination, Please try again." + } + }, + getStreamDifference: async ( + jobId: string, + streamsConfig: string, + ): Promise> => { + try { + const response = await api.post< + APIResponse<{ difference_streams: StreamsDataStructure }> + >( + `${API_CONFIG.ENDPOINTS.JOBS(API_CONFIG.PROJECT_ID)}/${jobId}/stream-difference`, + { updated_streams_config: streamsConfig }, + ) + return response.data + } catch (error) { + console.error("Error getting stream difference:", error) + throw error + } + }, } diff --git a/ui/src/api/services/sourceService.ts b/ui/src/api/services/sourceService.ts index 3f4d3a10..d36c9d64 100644 --- a/ui/src/api/services/sourceService.ts +++ b/ui/src/api/services/sourceService.ts @@ -79,7 +79,7 @@ export const sourceService = { version: source.version, config: source.config, }, - { timeout: 0 }, + { timeout: 0 }, // Disable timeout for this request since it can take longer ) return { success: response.data.success, @@ -101,7 +101,7 @@ export const sourceService = { const response = await api.get>( `${API_CONFIG.ENDPOINTS.SOURCES(API_CONFIG.PROJECT_ID)}/versions/?type=${type}`, { - timeout: 0, + timeout: 0, // Disable timeout for this request since it can take longer }, ) return response.data @@ -123,7 +123,7 @@ export const sourceService = { type: type.toLowerCase(), version, }, - { timeout: 300000, signal }, + { timeout: 300000, signal }, //timeout is 300000 as spec takes more time as it needs to fetch the spec from olake ) return response.data } catch (error) { @@ -132,6 +132,7 @@ export const sourceService = { } }, + //fetches source specific streams getSourceStreams: async ( name: string, type: string, diff --git a/ui/src/modules/auth/pages/Login.tsx b/ui/src/modules/auth/pages/Login.tsx index 7682fb55..e1cedc18 100644 --- a/ui/src/modules/auth/pages/Login.tsx +++ b/ui/src/modules/auth/pages/Login.tsx @@ -1,7 +1,7 @@ import { useState } from "react" import { useNavigate } from "react-router-dom" import { Form, Input, Button, Card, message } from "antd" -import { User, LockKey } from "@phosphor-icons/react" +import { UserIcon, LockKeyIcon } from "@phosphor-icons/react" import { useAppStore } from "../../../store" import { LoginArgs } from "../../../types" @@ -25,6 +25,7 @@ const Login: React.FC = () => { duration: 3, className: "font-medium", }) + // clear form in the case of error form.resetFields() } setLoading(false) @@ -56,7 +57,7 @@ const Login: React.FC = () => { > { > { centered >
- diff --git a/ui/src/modules/common/Modals/ClearDestinationAndSyncModal.tsx b/ui/src/modules/common/Modals/ClearDestinationAndSyncModal.tsx deleted file mode 100644 index 189083f1..00000000 --- a/ui/src/modules/common/Modals/ClearDestinationAndSyncModal.tsx +++ /dev/null @@ -1,55 +0,0 @@ -import { useNavigate } from "react-router-dom" -import { Warning } from "@phosphor-icons/react" -import { Button, message, Modal } from "antd" -import { useAppStore } from "../../../store" - -const ClearDestinationAndSyncModal = () => { - const { - showClearDestinationAndSyncModal, - setShowClearDestinationAndSyncModal, - } = useAppStore() - const navigate = useNavigate() - - return ( - -
- - -
- Clear destination and sync deletes all the data in your destination - and sync your job -
- -
- - -
-
- - ) -} - -export default ClearDestinationAndSyncModal diff --git a/ui/src/modules/common/Modals/ClearDestinationModal.tsx b/ui/src/modules/common/Modals/ClearDestinationModal.tsx new file mode 100644 index 00000000..1ad95370 --- /dev/null +++ b/ui/src/modules/common/Modals/ClearDestinationModal.tsx @@ -0,0 +1,83 @@ +import { useNavigate } from "react-router-dom" +import { WarningIcon } from "@phosphor-icons/react" +import { Button, message, Modal } from "antd" +import { useAppStore } from "../../../store" +import { jobService } from "../../../api" +import { useState } from "react" + +const ClearDestinationModal = () => { + const { + showClearDestinationModal, + setShowClearDestinationModal, + selectedJobId, + fetchJobs, + } = useAppStore() + const navigate = useNavigate() + + const [isLoading, setIsLoading] = useState(false) + + const handleClearDestination = async () => { + if (!selectedJobId) { + message.error("No job selected") + return + } + setIsLoading(true) + try { + const response = await jobService.clearDestination(selectedJobId) + // wait for 1 second before refreshing jobs to avoid fetching old state + await new Promise(resolve => setTimeout(resolve, 1000)) + await fetchJobs() + message.success(response.data.message) + navigate(`/jobs/${selectedJobId}/history`) + } catch (error) { + message.destroy() + message.error(error as string) + console.error("Failed to clear destination", error) + } finally { + setShowClearDestinationModal(false) + setIsLoading(false) + } + } + + return ( + +
+ + +
+ This will erase all data that was synced by this job in the + destination. This action{" "} + cannot be undone. Are you sure you + want to proceed? +
+ +
+ + +
+
+ + ) +} + +export default ClearDestinationModal diff --git a/ui/src/modules/common/Modals/DeleteJobModal.tsx b/ui/src/modules/common/Modals/DeleteJobModal.tsx index 8ac1513f..b040c45d 100644 --- a/ui/src/modules/common/Modals/DeleteJobModal.tsx +++ b/ui/src/modules/common/Modals/DeleteJobModal.tsx @@ -1,6 +1,6 @@ import { useNavigate } from "react-router-dom" import { Button, message, Modal } from "antd" -import { Warning } from "@phosphor-icons/react" +import { WarningIcon } from "@phosphor-icons/react" import { useAppStore } from "../../../store" @@ -25,7 +25,7 @@ const DeleteJobModal = ({ centered >
- diff --git a/ui/src/modules/common/Modals/DeleteModal.tsx b/ui/src/modules/common/Modals/DeleteModal.tsx index 696a51c3..f2ed251c 100644 --- a/ui/src/modules/common/Modals/DeleteModal.tsx +++ b/ui/src/modules/common/Modals/DeleteModal.tsx @@ -1,12 +1,13 @@ import { formatDistanceToNow } from "date-fns" import { Button, message, Modal, Table } from "antd" -import { Warning } from "@phosphor-icons/react" +import { WarningIcon } from "@phosphor-icons/react" import { useAppStore } from "../../../store" import { Entity } from "../../../types" import { DeleteModalProps } from "../../../types/modalTypes" import { getConnectorImage } from "../../../utils/utils" +//Entity Delete Modal const DeleteModal = ({ fromSource }: DeleteModalProps) => { const { showDeleteModal, @@ -137,7 +138,7 @@ const DeleteModal = ({ fromSource }: DeleteModalProps) => { width={600} >
- diff --git a/ui/src/modules/common/Modals/DestinationDatabaseModal.tsx b/ui/src/modules/common/Modals/DestinationDatabaseModal.tsx index 03a92e86..20e51655 100644 --- a/ui/src/modules/common/Modals/DestinationDatabaseModal.tsx +++ b/ui/src/modules/common/Modals/DestinationDatabaseModal.tsx @@ -8,7 +8,7 @@ import { LABELS, NAMESPACE_PLACEHOLDER, } from "../../../utils/constants" -import { DotOutline } from "@phosphor-icons/react" +import { DotOutlineIcon } from "@phosphor-icons/react" import { DestinationDatabaseModalProps } from "../../../types" type FormatType = (typeof FORMAT_OPTIONS)[keyof typeof FORMAT_OPTIONS] @@ -177,7 +177,7 @@ const DestinationDatabaseModal = ({ key={index} className="flex items-center text-sm" > - diff --git a/ui/src/modules/common/Modals/EditSourceModal.tsx b/ui/src/modules/common/Modals/EditSourceModal.tsx deleted file mode 100644 index 2a492a0d..00000000 --- a/ui/src/modules/common/Modals/EditSourceModal.tsx +++ /dev/null @@ -1,199 +0,0 @@ -import { Button, Modal, Table, message } from "antd" -import { CheckCircle, Warning } from "@phosphor-icons/react" -import { formatDistanceToNow } from "date-fns" -import { useNavigate } from "react-router-dom" - -import { sourceService } from "../../../api" -import { useAppStore } from "../../../store" -import { getConnectorImage } from "../../../utils/utils" - -const EditSourceModal = () => { - const navigate = useNavigate() - const { - showEditSourceModal, - setShowEditSourceModal, - showSuccessModal, - setShowSuccessModal, - selectedSource, - updateSource, - setShowTestingModal, - setShowFailureModal, - setSourceTestConnectionError, - } = useAppStore() - - const getSourceData = () => { - const configStr = - typeof selectedSource?.config === "string" - ? selectedSource?.config - : JSON.stringify(selectedSource?.config) - - const sourceData = { - name: selectedSource?.name, - type: selectedSource?.type, - version: selectedSource?.version, - config: configStr, - } - return sourceData - } - - const handleEdit = async () => { - if (!selectedSource?.id) { - message.error("Source ID is missing") - return - } - - try { - setShowEditSourceModal(false) - setShowTestingModal(true) - const testResult = - await sourceService.testSourceConnection(getSourceData()) - - if (testResult.data?.status === "SUCCEEDED") { - setTimeout(() => { - setShowTestingModal(false) - setShowSuccessModal(true) - }, 1000) - - setTimeout(async () => { - setShowSuccessModal(false) - await updateSource(selectedSource.id.toString(), selectedSource) - navigate("/sources") - }, 2000) - } else { - setShowTestingModal(false) - setSourceTestConnectionError(testResult.data?.message || "") - setShowFailureModal(true) - } - } catch (error) { - message.error("Failed to update source") - console.error(error) - } - } - - return ( - <> - - -
- } - open={showEditSourceModal} - onCancel={() => setShowEditSourceModal(false)} - footer={[ - , - , - ]} - centered - width="38%" - > -
-

- Due to the editing, the jobs are going to get affected -

-

- Editing this source will affect the following jobs that are - associated with this source and as a result will fail immediately. - Do you still want to edit the source? -

-
-
- ( - - {activate ? "Active" : "Inactive"} - - ), - }, - { - title: "Last runtime", - dataIndex: "last_run_time", - key: "last_run_time", - render: (text: string) => ( - - {text !== undefined - ? formatDistanceToNow(new Date(text), { - addSuffix: true, - }) - : "-"} - - ), - }, - { - title: "Destination", - dataIndex: "destination_name", - key: "destination_name", - render: (destination_name: string, record: any) => ( -
- {record.destination_type} - {destination_name || "N/A"} -
- ), - }, - ]} - dataSource={selectedSource?.jobs} - pagination={false} - rowKey="key" - scroll={{ y: 300 }} - /> - - - - {/* Success Modal */} - -
- -
- Changes are saved successfully -
-
- - - ) -} - -export default EditSourceModal diff --git a/ui/src/modules/common/Modals/EntityCancelModal.tsx b/ui/src/modules/common/Modals/EntityCancelModal.tsx index 3739fcb0..22e2f2ef 100644 --- a/ui/src/modules/common/Modals/EntityCancelModal.tsx +++ b/ui/src/modules/common/Modals/EntityCancelModal.tsx @@ -1,7 +1,11 @@ import React from "react" import { useNavigate } from "react-router-dom" import { Button, Modal } from "antd" -import { GitCommit, LinktreeLogo, Path } from "@phosphor-icons/react" +import { + GitCommitIcon, + LinktreeLogoIcon, + PathIcon, +} from "@phosphor-icons/react" import { useAppStore } from "../../../store" import { JOB_CREATION_STEPS } from "../../../utils/constants" @@ -28,11 +32,11 @@ const EntityCancelModal: React.FC = ({
{type === JOB_CREATION_STEPS.SOURCE ? ( - + ) : type === JOB_CREATION_STEPS.DESTINATION ? ( - + ) : ( - + )}
diff --git a/ui/src/modules/common/Modals/EntityEditModal.tsx b/ui/src/modules/common/Modals/EntityEditModal.tsx index 74d31bbe..a3712d4c 100644 --- a/ui/src/modules/common/Modals/EntityEditModal.tsx +++ b/ui/src/modules/common/Modals/EntityEditModal.tsx @@ -1,13 +1,14 @@ import { useNavigate } from "react-router-dom" import { formatDistanceToNow } from "date-fns" import { Button, Modal, Table, message } from "antd" -import { InfoIcon, Warning } from "@phosphor-icons/react" +import { InfoIcon, WarningIcon } from "@phosphor-icons/react" import { useAppStore } from "../../../store" import { sourceService } from "../../../api" import { destinationService } from "../../../api/services/destinationService" import { EntityEditModalProps } from "../../../types" import { getConnectorImage } from "../../../utils/utils" +import { TEST_CONNECTION_STATUS } from "../../../utils/constants" const EntityEditModal = ({ entityType }: EntityEditModalProps) => { const navigate = useNavigate() @@ -69,7 +70,10 @@ const EntityEditModal = ({ entityType }: EntityEditModalProps) => { ? await sourceService.testSourceConnection(getEntityData()) : await destinationService.testDestinationConnection(getEntityData()) - if (testResult.data?.status === "SUCCEEDED") { + if ( + testResult.data?.connection_result.status === + TEST_CONNECTION_STATUS.SUCCEEDED + ) { setTimeout(() => { setShowTestingModal(false) setShowSuccessModal(true) @@ -82,8 +86,12 @@ const EntityEditModal = ({ entityType }: EntityEditModalProps) => { navigate(navigatePath) }, 2000) } else { + const testConnectionError = { + message: testResult.data?.connection_result.message || "", + logs: testResult.data?.logs || [], + } setShowTestingModal(false) - setTestConnectionError(testResult.data?.message || "") + setTestConnectionError(testConnectionError) setShowFailureModal(true) } } catch (error) { @@ -157,7 +165,7 @@ const EntityEditModal = ({ entityType }: EntityEditModalProps) => { - diff --git a/ui/src/modules/common/Modals/EntitySavedModal.tsx b/ui/src/modules/common/Modals/EntitySavedModal.tsx index fa646348..5d458c8f 100644 --- a/ui/src/modules/common/Modals/EntitySavedModal.tsx +++ b/ui/src/modules/common/Modals/EntitySavedModal.tsx @@ -1,5 +1,10 @@ import { useNavigate } from "react-router-dom" -import { Check, GitCommit, Path, LinktreeLogo } from "@phosphor-icons/react" +import { + CheckIcon, + GitCommitIcon, + PathIcon, + LinktreeLogoIcon, +} from "@phosphor-icons/react" import { Button, Modal } from "antd" import { useAppStore } from "../../../store" import { EntitySavedModalProps } from "../../../types" @@ -23,11 +28,11 @@ const EntitySavedModal: React.FC = ({
{type === "source" ? ( - + ) : type === JOB_CREATION_STEPS.STREAMS ? ( - + ) : ( - + )}
@@ -40,11 +45,11 @@ const EntitySavedModal: React.FC = ({
{type === "source" ? ( - + ) : type === JOB_CREATION_STEPS.STREAMS ? ( - + ) : ( - + )} {entityName || @@ -56,7 +61,7 @@ const EntitySavedModal: React.FC = ({
- + Success
diff --git a/ui/src/modules/common/Modals/IngestionModeChangeModal.tsx b/ui/src/modules/common/Modals/IngestionModeChangeModal.tsx new file mode 100644 index 00000000..4a3ebab1 --- /dev/null +++ b/ui/src/modules/common/Modals/IngestionModeChangeModal.tsx @@ -0,0 +1,56 @@ +import { Button, Modal } from "antd" +import { useAppStore } from "../../../store" +import { IngestionModeChangeModalProps } from "../../../types/modalTypes" + +const IngestionModeChangeModal = ({ + onConfirm, + ingestionMode, +}: IngestionModeChangeModalProps) => { + const { showIngestionModeChangeModal, setShowIngestionModeChangeModal } = + useAppStore() + + return ( + +
+
+ Switch to {ingestionMode} for all tables ? +
+ +
+
+ All tables will be switched to {ingestionMode} mode, +
+
+ You can change mode for individual tables +
+
+ +
+ + +
+
+ + ) +} + +export default IngestionModeChangeModal diff --git a/ui/src/modules/common/Modals/StreamDifferenceModal.tsx b/ui/src/modules/common/Modals/StreamDifferenceModal.tsx new file mode 100644 index 00000000..8d9e7b04 --- /dev/null +++ b/ui/src/modules/common/Modals/StreamDifferenceModal.tsx @@ -0,0 +1,106 @@ +import { InfoIcon, WarningIcon } from "@phosphor-icons/react" +import { Button, Modal } from "antd" +import { useAppStore } from "../../../store" +import { StreamDifferenceModalProps } from "../../../types/modalTypes" + +const StreamDifferenceModal = ({ + streamDifference, + onConfirm, +}: StreamDifferenceModalProps) => { + const { showStreamDifferenceModal, setShowStreamDifferenceModal } = + useAppStore() + + const handleCloseModal = () => { + setShowStreamDifferenceModal(false) + } + + const handleFinish = () => { + onConfirm() + setShowStreamDifferenceModal(false) + } + + const renderStreamsByNamespace = () => { + const namespaces = Object.keys(streamDifference.selected_streams) + + return namespaces.map(namespace => { + const streams = streamDifference.selected_streams[namespace] + + if (!streams || streams.length === 0) return null + + return ( +
    +
  • + {namespace} +
      + {streams.map((stream, index) => ( +
    • + {stream.stream_name} +
      • + ))} +
    +
    • +
+ ) + }) + } + + return ( + <> + + +
+ } + open={showStreamDifferenceModal} + onCancel={handleCloseModal} + footer={[ + , + , + ]} + centered + width="30%" + > +
+

+ Are you sure you want to continue? +

+

+ Modifying stream configurations will clear destination data for the + impacted streams. Following streams will be impacted: +

+
+ + Any ongoing sync will be auto cancelled. +
+
+
+ {renderStreamsByNamespace()} +
+ + + ) +} + +export default StreamDifferenceModal diff --git a/ui/src/modules/common/Modals/StreamEditDisabledModal.tsx b/ui/src/modules/common/Modals/StreamEditDisabledModal.tsx new file mode 100644 index 00000000..696a89ec --- /dev/null +++ b/ui/src/modules/common/Modals/StreamEditDisabledModal.tsx @@ -0,0 +1,58 @@ +import { InfoIcon } from "@phosphor-icons/react" +import { Button, Modal } from "antd" +import { useAppStore } from "../../../store" +import { useNavigate } from "react-router-dom" +import { StreamEditDisabledModalProps } from "../../../types/modalTypes" + +const StreamEditDisabledModal = ({ from }: StreamEditDisabledModalProps) => { + const navigate = useNavigate() + const { showStreamEditDisabledModal, setShowStreamEditDisabledModal } = + useAppStore() + + const handleCloseModal = () => { + navigate("/jobs") + setShowStreamEditDisabledModal(false) + } + + return ( + <> + + +
+ } + open={showStreamEditDisabledModal} + closable={false} + footer={ +
+ +
+ } + centered + width="30%" + > +
+

Editing Disabled

+

+ {from === "jobSettings" ? "Job Settings Edit" : "Stream editing"} is + disabled while the destination is being cleared. It will be + available once the process finishes. +

+
+ + + ) +} + +export default StreamEditDisabledModal diff --git a/ui/src/modules/common/Modals/TestConnectionFailureModal.tsx b/ui/src/modules/common/Modals/TestConnectionFailureModal.tsx index 402ca57b..5056cc10 100644 --- a/ui/src/modules/common/Modals/TestConnectionFailureModal.tsx +++ b/ui/src/modules/common/Modals/TestConnectionFailureModal.tsx @@ -1,9 +1,12 @@ +import { useState } from "react" import { useNavigate } from "react-router-dom" -import { Modal } from "antd" -import { Info } from "@phosphor-icons/react" +import { message, Modal } from "antd" +import { CopySimpleIcon } from "@phosphor-icons/react" +import clsx from "clsx" import { useAppStore } from "../../../store" import ErrorIcon from "../../../assets/ErrorIcon.svg" +import { getLogTextColor, getLogLevelClass } from "../../../utils/utils" const TestConnectionFailureModal = ({ fromSources, @@ -16,14 +19,17 @@ const TestConnectionFailureModal = ({ sourceTestConnectionError, destinationTestConnectionError, } = useAppStore() + const [isExpanded, setIsExpanded] = useState(false) const navigate = useNavigate() const handleCancel = () => { setShowFailureModal(false) + setIsExpanded(false) } const handleBackToPath = () => { setShowFailureModal(false) + setIsExpanded(false) if (fromSources) { navigate("/sources") } else { @@ -31,15 +37,39 @@ const TestConnectionFailureModal = ({ } } + const handleReadMore = () => setIsExpanded(!isExpanded) + + const handleCopyLogs = async () => { + try { + await navigator.clipboard.writeText( + JSON.stringify( + fromSources + ? sourceTestConnectionError?.logs || [] + : destinationTestConnectionError?.logs || [], + null, + 4, + ), + ) + message.success("Logs copied to clipboard!") + } catch { + message.error("Failed to copy logs") + } + } + return ( -
+
-
+

Failed

-

+

Your test connection has failed

-
- - - {fromSources - ? sourceTestConnectionError - : destinationTestConnectionError} - +
+
+
Error
+ {isExpanded && ( + + )} +
+
+ {!isExpanded ? ( +
+ {fromSources + ? sourceTestConnectionError?.message || "" + : destinationTestConnectionError?.message || ""} +
+ ) : ( +
+ + {(fromSources + ? sourceTestConnectionError?.logs || [] + : destinationTestConnectionError?.logs || [] + ).map((jobLog, index) => ( + + + + + ))} + +
+ + {jobLog.level} + + + {jobLog.message} +
+ )} + + {!isExpanded && ( + + )} +
diff --git a/ui/src/modules/common/components/DocumentationPanel.tsx b/ui/src/modules/common/components/DocumentationPanel.tsx index b4c008f1..5021d21e 100644 --- a/ui/src/modules/common/components/DocumentationPanel.tsx +++ b/ui/src/modules/common/components/DocumentationPanel.tsx @@ -2,10 +2,10 @@ import { useState, useRef, useEffect } from "react" import clsx from "clsx" import { Button, Tooltip } from "antd" import { - CornersOut, - CaretRight, - Info, - ArrowSquareOut, + CornersOutIcon, + CaretRightIcon, + InfoIcon, + ArrowSquareOutIcon, } from "@phosphor-icons/react" import { DocumentationPanelProps } from "../../../types" @@ -44,6 +44,7 @@ const DocumentationPanel: React.FC = ({ if (!iframe) return const handleLoad = () => { + // as the theme for ui is light themed we need to show only light theme in docs website as the default theme is dark // Post message to iframe for theming iframe.contentWindow?.postMessage({ theme: "light" }, "https://olake.io") @@ -78,7 +79,7 @@ const DocumentationPanel: React.FC = ({ className="flex items-center" onClick={openInNewTab} icon={ - @@ -91,7 +92,7 @@ const DocumentationPanel: React.FC = ({ className="flex items-center bg-blue-600" onClick={toggleDocPanel} icon={ - @@ -120,7 +121,7 @@ const DocumentationPanel: React.FC = ({ isDocPanelCollapsed ? "rotate-180" : "rotate-0", )} > - +
@@ -144,7 +145,7 @@ const DocumentationPanel: React.FC = ({ diff --git a/ui/src/modules/common/components/Form/BooleanSwitchWidget.tsx b/ui/src/modules/common/components/Form/BooleanSwitchWidget.tsx index 36338023..c50a622a 100644 --- a/ui/src/modules/common/components/Form/BooleanSwitchWidget.tsx +++ b/ui/src/modules/common/components/Form/BooleanSwitchWidget.tsx @@ -1,3 +1,6 @@ +/** + * BooleanSwitchWidget is a component that renders a boolean switch this overrides the default boolean field template of rjsf + */ import { Switch } from "antd" import { WidgetProps } from "@rjsf/utils" diff --git a/ui/src/modules/common/components/Form/CustomFieldTemplate.tsx b/ui/src/modules/common/components/Form/CustomFieldTemplate.tsx index 85e384ba..0348f1fe 100644 --- a/ui/src/modules/common/components/Form/CustomFieldTemplate.tsx +++ b/ui/src/modules/common/components/Form/CustomFieldTemplate.tsx @@ -1,5 +1,5 @@ import { FieldTemplateProps } from "@rjsf/utils" -import { Info, Plus, Trash } from "@phosphor-icons/react" +import { InfoIcon, PlusIcon, TrashIcon } from "@phosphor-icons/react" import { Tooltip, Button } from "antd" import { useState, useEffect } from "react" @@ -42,13 +42,13 @@ function KeyValueRow({
- - @@ -123,7 +129,7 @@ const Sidebar: React.FC<{ collapsed ? "rotate-180" : "rotate-0", )} > - +
diff --git a/ui/src/modules/destinations/components/DestinationEmptyState.tsx b/ui/src/modules/destinations/components/DestinationEmptyState.tsx index 3af708eb..f2fa1d10 100644 --- a/ui/src/modules/destinations/components/DestinationEmptyState.tsx +++ b/ui/src/modules/destinations/components/DestinationEmptyState.tsx @@ -1,5 +1,5 @@ import { Button } from "antd" -import { PlayCircle, Plus } from "@phosphor-icons/react" +import { PlayCircleIcon, PlusIcon } from "@phosphor-icons/react" import { DestinationTutorialYTLink } from "../../../utils/constants" import FirstDestination from "../../../assets/FirstDestination.svg" @@ -29,7 +29,7 @@ const DestinationEmptyState = ({ className="border-1 mb-12 border-[1px] border-[#D9D9D9] bg-white px-6 py-4 text-black" onClick={handleCreateDestination} > - + New Destination
@@ -48,7 +48,7 @@ const DestinationEmptyState = ({
- + OLake/ Tutorial
diff --git a/ui/src/modules/destinations/components/DestinationTable.tsx b/ui/src/modules/destinations/components/DestinationTable.tsx index f2b25d8e..b8d0e97e 100644 --- a/ui/src/modules/destinations/components/DestinationTable.tsx +++ b/ui/src/modules/destinations/components/DestinationTable.tsx @@ -1,6 +1,10 @@ import { useState } from "react" import { Table, Input, Button, Dropdown, Pagination } from "antd" -import { DotsThree, PencilSimpleLine, Trash } from "@phosphor-icons/react" +import { + DotsThreeIcon, + PencilSimpleLineIcon, + TrashIcon, +} from "@phosphor-icons/react" import { DestinationTableProps, Entity } from "../../../types" import { getConnectorImage } from "../../../utils/utils" @@ -31,13 +35,13 @@ const DestinationTable: React.FC = ({ items: [ { key: "edit", - icon: , + icon: , label: "Edit", onClick: () => onEdit(String(record.id)), }, { key: "delete", - icon: , + icon: , label: "Delete", danger: true, onClick: () => onDelete(record), @@ -49,7 +53,7 @@ const DestinationTable: React.FC = ({ >
), }, @@ -26,7 +26,7 @@ export const connectorOptions: ConnectorOption[] = [ alt="Apache Iceberg" className="mr-2 size-5" /> - Apache Iceberg + Apache Iceberg
), }, diff --git a/ui/src/modules/destinations/pages/CreateDestination.tsx b/ui/src/modules/destinations/pages/CreateDestination.tsx index 43a2069a..b54e137d 100644 --- a/ui/src/modules/destinations/pages/CreateDestination.tsx +++ b/ui/src/modules/destinations/pages/CreateDestination.tsx @@ -35,6 +35,7 @@ import { DESTINATION_INTERNAL_TYPES, OLAKE_LATEST_VERSION_URL, SETUP_TYPES, + TEST_CONNECTION_STATUS, transformErrors, } from "../../../utils/constants" import EndpointTitle from "../../../utils/EndpointTitle" @@ -267,6 +268,7 @@ const CreateDestination = forwardRef< if (setupType === SETUP_TYPES.EXISTING) return setLoading(true) + // cancels old requests when new one is made return withAbortController( signal => destinationService.getDestinationSpec( @@ -306,6 +308,7 @@ const CreateDestination = forwardRef< setShowSourceCancelModal(true) } + //makes sure user enters destination name and version and fills all the required fields in the form const validateDestination = async (): Promise => { try { if (setupType === SETUP_TYPES.NEW) { @@ -368,11 +371,15 @@ const CreateDestination = forwardRef< try { setShowTestingModal(true) + //test the connection and show either success or failure modal based on the result const testResult = await destinationService.testDestinationConnection(newDestinationData) setShowTestingModal(false) - if (testResult.data?.status === "SUCCEEDED") { + if ( + testResult.data?.connection_result.status === + TEST_CONNECTION_STATUS.SUCCEEDED + ) { setShowSuccessModal(true) setTimeout(() => { setShowSuccessModal(false) @@ -381,7 +388,11 @@ const CreateDestination = forwardRef< .catch(error => console.error("Error adding destination:", error)) }, 1000) } else { - setDestinationTestConnectionError(testResult.data?.message || "") + const testConnectionError = { + message: testResult.data?.connection_result.message || "", + logs: testResult.data?.logs || [], + } + setDestinationTestConnectionError(testConnectionError) setShowFailureModal(true) } } catch (error) { @@ -493,6 +504,7 @@ const CreateDestination = forwardRef<
= ({ {loadingVersions ? ( @@ -527,6 +537,7 @@ const DestinationEdit: React.FC = ({ ) : versions.length > 0 ? (
) : availableVersions.length > 0 ? (