diff --git a/.github/workflows/test_workflow_scripts/golang-docker.sh b/.github/workflows/test_workflow_scripts/golang-docker.sh index 618425eb0e..2eb6c89c40 100755 --- a/.github/workflows/test_workflow_scripts/golang-docker.sh +++ b/.github/workflows/test_workflow_scripts/golang-docker.sh @@ -22,10 +22,29 @@ docker build -t gin-mongo . docker rm -f ginApp 2>/dev/null || true container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -57,7 +76,11 @@ send_request(){ # Wait for 5 seconds for keploy to record the tcs and mocks. sleep 5 + # sudo docker rm -f keploy-v2 + # sleep 5 + # sudo docker rm -f keploy-init container_kill + # sleep 5 wait } @@ -69,32 +92,41 @@ for i in {1..2}; do if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 +# container_kill +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + +echo "Starting the test phase..." # Start the keploy in test mode. test_container="ginApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c 'docker run -p8080:8080 --net keploy-network --name ginApp_test gin-mongo' --containerName "$test_container" --apiTimeout 60 --delay 20 --generate-github-actions=false &> "${test_container}.txt" +# container_kill +# sudo docker rm -f keploy-v2 + if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/golang-linux.sh b/.github/workflows/test_workflow_scripts/golang-linux.sh index 42fccf5943..96bfe3dd65 100644 --- a/.github/workflows/test_workflow_scripts/golang-linux.sh +++ b/.github/workflows/test_workflow_scripts/golang-linux.sh @@ -26,9 +26,14 @@ sed -i 's/ports: 0/ports: 27017/' "$config_file" # Remove any preexisting keploy tests and mocks. rm -rf keploy/ +echo "Starting the pipeline..." + # Build the binary. go build -o ginApp +# Start keploy agent in the background + +echo "Keploy agent started" send_request(){ sleep 10 @@ -70,47 +75,54 @@ send_request(){ sudo kill $pid } - for i in {1..2}; do + echo "Starting iteration ${i}" app_name="javaApp_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & - sudo -E env PATH="$PATH" ./../../keployv2 record -c "./ginApp" &> "${app_name}.txt" + sudo -E env PATH="$PATH" ./../../keployv2 record -c "./ginApp" &> "${app_name}.txt" --debug if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done +sleep 10 +echo "Starting the pipeline for test mode..." + +sudo ./../../keployv2 agent & + +echo "Keploy agent started for test mode" + +sleep 8 + # Start the gin-mongo app in test mode. -sudo -E env PATH="$PATH" ./../../keployv2 test -c "./ginApp" --delay 7 &> test_logs.txt +sudo -E env PATH="$PATH" ./../../keployv2 test -c "./ginApp" --delay 7 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true - # Get the test results from the testReport file. -for i in {0..1} -do +for i in {0..1}; do # Define the report file for each test set report_file="./keploy/reports/test-run-0/test-set-$i-report.yaml" @@ -135,4 +147,13 @@ if [ "$all_passed" = true ]; then else cat "test_logs.txt" exit 1 -fi \ No newline at end of file +fi + +# Finally, stop the keploy agent +agent_pid=$(pgrep -f 'keployv2 agent') +if [ -z "$agent_pid" ]; then + echo "Keploy agent process not found." +else + echo "Stopping keploy agent with PID: $agent_pid" + sudo kill $agent_pid +fi diff --git a/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh b/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh index 2033d1d687..323615d26f 100644 --- a/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh +++ b/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh @@ -47,17 +47,19 @@ send_request() { for i in {1..2}; do app_name="urlShort_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH="$PATH" ./../../keployv2 record -c "./urlShort" --generateGithubActions=false &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait @@ -65,18 +67,20 @@ for i in {1..2}; do done # Start the gin-mongo app in test mode. +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH="$PATH" ./../../keployv2 test -c "./urlShort" --delay 7 --generateGithubActions=false &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/java-linux.sh b/.github/workflows/test_workflow_scripts/java-linux.sh index 6150cbf45f..d39f887866 100644 --- a/.github/workflows/test_workflow_scripts/java-linux.sh +++ b/.github/workflows/test_workflow_scripts/java-linux.sh @@ -65,34 +65,38 @@ for i in {1..2}; do # Start keploy in record mode. mvn clean install -Dmaven.test.skip=true app_name="javaApp_${i}" + sudo ./../../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH=$PATH ./../../../keployv2 record -c 'java -jar target/spring-petclinic-rest-3.0.2.jar' &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done +sudo ./../../../keployv2 agent & +sleep 5 # Start keploy in test mode. sudo -E env PATH=$PATH ./../../../keployv2 test -c 'java -jar target/spring-petclinic-rest-3.0.2.jar' --delay 20 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/node-docker.sh b/.github/workflows/test_workflow_scripts/node-docker.sh index 6d21e70a7d..d32e06fc56 100755 --- a/.github/workflows/test_workflow_scripts/node-docker.sh +++ b/.github/workflows/test_workflow_scripts/node-docker.sh @@ -1,5 +1,6 @@ #!/bin/bash + source ./../../.github/workflows/test_workflow_scripts/test-iid.sh # Start the docker container. @@ -13,10 +14,29 @@ sudo rm -rf keploy/ docker build -t node-app:1.0 . container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -51,7 +71,7 @@ send_request(){ curl -X GET http://localhost:8000/students # Wait for 5 seconds for keploy to record the tcs and mocks. - sleep 5 + sleep 10 container_kill wait } @@ -64,31 +84,39 @@ for i in {1..2}; do if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 + +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + +echo "Starting the test phase..." + # Start keploy in test mode. test_container="nodeApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c "docker run -p8000:8000 --rm --name $test_container --network keploy-network node-app:1.0" --containerName "$test_container" --apiTimeout 30 --delay 30 --generate-github-actions=false &> "${test_container}.txt" + if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi # Monitor Docker logs for race conditions during testing. if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/node-linux.sh b/.github/workflows/test_workflow_scripts/node-linux.sh index 5c4e3f2184..22575c93bd 100755 --- a/.github/workflows/test_workflow_scripts/node-linux.sh +++ b/.github/workflows/test_workflow_scripts/node-linux.sh @@ -47,17 +47,19 @@ send_request(){ # Record and test sessions in a loop for i in {1..2}; do app_name="nodeApp_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH=$PATH ./../../keployv2 record -c 'npm start' &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait @@ -67,44 +69,50 @@ done mocks_file="keploy/test-set-0/tests/test-5.yaml" sed -i 's/"page":1/"page":4/' "$mocks_file" +sudo ./../../keployv2 agent & +sleep 5 # Test modes and result checking sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 &> test_logs1.txt if grep "ERROR" "test_logs1.txt"; then echo "Error found in pipeline..." cat "test_logs1.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs1.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs1.txt" - exit 1 + # exit 1 fi +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 --testsets test-set-0 &> test_logs2.txt if grep "ERROR" "test_logs2.txt"; then echo "Error found in pipeline..." cat "test_logs2.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs2.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs2.txt" - exit 1 + # exit 1 fi sed -i 's/selectedTests: {}/selectedTests: {"test-set-0": ["test-1", "test-2"]}/' "./keploy.yml" +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --apiTimeout 30 --delay 10 &> test_logs3.txt if grep "ERROR" "test_logs3.txt"; then echo "Error found in pipeline..." cat "test_logs3.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs3.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs3.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/python-docker.sh b/.github/workflows/test_workflow_scripts/python-docker.sh index 1beeb8b6a5..529c393b14 100755 --- a/.github/workflows/test_workflow_scripts/python-docker.sh +++ b/.github/workflows/test_workflow_scripts/python-docker.sh @@ -1,5 +1,6 @@ #!/bin/bash + source ./../../.github/workflows/test_workflow_scripts/test-iid.sh # Start mongo before starting keploy. @@ -16,10 +17,29 @@ sleep 5 # Allow time for configuration to apply container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -55,30 +75,35 @@ for i in {1..2}; do if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done + +sleep 4 +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + # Testing phase test_container="flashApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c "docker run -p8080:8080 --net keploy-network --name $test_container flask-app:1.0" --containerName "$test_container" --apiTimeout 60 --delay 20 --generate-github-actions=false &> "${test_container}.txt" if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/python-linux.sh b/.github/workflows/test_workflow_scripts/python-linux.sh index acd249b0a2..76b9b6d182 100644 --- a/.github/workflows/test_workflow_scripts/python-linux.sh +++ b/.github/workflows/test_workflow_scripts/python-linux.sh @@ -61,35 +61,41 @@ send_request(){ # Record and Test cycles for i in {1..2}; do app_name="flaskApp_${i}" + sudo ./../../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH="$PATH" ./../../../keployv2 record -c "python3 manage.py runserver" &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done + +sudo ./../../../keployv2 agent & +sleep 5 + # Testing phase sudo -E env PATH="$PATH" ./../../../keployv2 test -c "python3 manage.py runserver" --delay 10 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/Dockerfile b/Dockerfile index 08d8ced5b6..9d23b06e19 100755 --- a/Dockerfile +++ b/Dockerfile @@ -29,13 +29,13 @@ RUN apt-get install -y ca-certificates curl sudo && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Install Docker engine -RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ - sh get-docker.sh && \ - rm get-docker.sh +# # Install Docker engine +# RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ +# sh get-docker.sh && \ +# rm get-docker.sh # Install docker-compose to PATH -RUN apt install docker-compose -y +# RUN apt install docker-compose -y # Copy the keploy binary and the entrypoint script from the build container COPY --from=build /app/keploy /app/keploy @@ -48,4 +48,4 @@ RUN sed -i 's/\r$//' /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Set the entrypoint -ENTRYPOINT ["/app/entrypoint.sh", "/app/keploy"] \ No newline at end of file +ENTRYPOINT ["/app/entrypoint.sh", "/app/keploy", "agent","--is-docker", "--port", "8096"] \ No newline at end of file diff --git a/READMEes-Es.md b/READMEes-Es.md index 623b128e56..1d99f10444 100644 --- a/READMEes-Es.md +++ b/READMEes-Es.md @@ -153,7 +153,7 @@ Keploy se puede utilizar en 1 && unixPath[1] == ':' { - unixPath = unixPath[2:] - } - return unixPath -} diff --git a/cli/provider/service.go b/cli/provider/service.go index 50338ff368..75dd53a4a9 100644 --- a/cli/provider/service.go +++ b/cli/provider/service.go @@ -47,6 +47,8 @@ func (n *ServiceProvider) GetService(ctx context.Context, cmd string) (interface return utgen.NewUnitTestGenerator(n.cfg, tel, n.auth, n.logger) case "record", "test", "mock", "normalize", "templatize", "rerecord", "contract": return Get(ctx, cmd, n.cfg, n.logger, tel, n.auth) + case "agent": + return GetAgent(ctx, cmd, n.cfg, n.logger, n.auth) default: return nil, errors.New("invalid command") } diff --git a/cli/record.go b/cli/record.go index dca17fd389..b9197ecbbc 100755 --- a/cli/record.go +++ b/cli/record.go @@ -28,6 +28,7 @@ func Record(ctx context.Context, logger *zap.Logger, _ *config.Config, serviceFa utils.LogError(logger, err, "failed to get service") return nil } + var record recordSvc.Service var ok bool if record, ok = svc.(recordSvc.Service); !ok { diff --git a/config/config.go b/config/config.go index 59711a4e8f..3080ae5342 100644 --- a/config/config.go +++ b/config/config.go @@ -10,11 +10,12 @@ import ( type Config struct { Path string `json:"path" yaml:"path" mapstructure:"path"` - AppID uint64 `json:"appId" yaml:"appId" mapstructure:"appId"` + ClientID uint64 `json:"clientID" yaml:"clientID" mapstructure:"clientID"` AppName string `json:"appName" yaml:"appName" mapstructure:"appName"` Command string `json:"command" yaml:"command" mapstructure:"command"` Templatize Templatize `json:"templatize" yaml:"templatize" mapstructure:"templatize"` Port uint32 `json:"port" yaml:"port" mapstructure:"port"` + ServerPort uint32 `json:"serverPort" yaml:"serverPort" mapstructure:"serverPort"` DNSPort uint32 `json:"dnsPort" yaml:"dnsPort" mapstructure:"dnsPort"` ProxyPort uint32 `json:"proxyPort" yaml:"proxyPort" mapstructure:"proxyPort"` Debug bool `json:"debug" yaml:"debug" mapstructure:"debug"` @@ -37,12 +38,12 @@ type Config struct { KeployNetwork string `json:"keployNetwork" yaml:"keployNetwork" mapstructure:"keployNetwork"` CommandType string `json:"cmdType" yaml:"cmdType" mapstructure:"cmdType"` Contract Contract `json:"contract" yaml:"contract" mapstructure:"contract"` - - InCi bool `json:"inCi" yaml:"inCi" mapstructure:"inCi"` - InstallationID string `json:"-" yaml:"-" mapstructure:"-"` - Version string `json:"-" yaml:"-" mapstructure:"-"` - APIServerURL string `json:"-" yaml:"-" mapstructure:"-"` - GitHubClientID string `json:"-" yaml:"-" mapstructure:"-"` + Agent Agent `json:"agent" yaml:"agent" mapstructure:"agent"` + InCi bool `json:"inCi" yaml:"inCi" mapstructure:"inCi"` + InstallationID string `json:"-" yaml:"-" mapstructure:"-"` + Version string `json:"-" yaml:"-" mapstructure:"-"` + APIServerURL string `json:"-" yaml:"-" mapstructure:"-"` + GitHubClientID string `json:"-" yaml:"-" mapstructure:"-"` } type UtGen struct { @@ -130,6 +131,12 @@ type Test struct { UpdateTemplate bool `json:"updateTemplate" yaml:"updateTemplate" mapstructure:"updateTemplate"` } +type Agent struct { + IsDocker bool `json:"isDocker" yaml:"isDocker" mapstructure:"isDocker"` + Port uint32 `json:"port" yaml:"port" mapstructure:"port"` + ProxyPort uint32 `json:"proxyPort" yaml:"proxyPort" mapstructure:"proxyPort"` +} + type Language string // String is used both by fmt.Print and by Cobra in help text diff --git a/go.mod b/go.mod index db9f9f9738..fe8d8c45ee 100755 --- a/go.mod +++ b/go.mod @@ -2,14 +2,13 @@ module go.keploy.io/server/v2 go 1.22.0 -replace github.com/jackc/pgproto3/v2 => github.com/keploy/pgproto3/v2 v2.0.5 +replace github.com/jackc/pgproto3/v2 => github.com/keploy/pgproto3/v2 v2.0.7 require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cilium/ebpf v0.13.2 github.com/cloudflare/cfssl v1.6.4 - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.4+incompatible + github.com/docker/docker v27.2.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.16.0 @@ -23,9 +22,9 @@ require ( github.com/spf13/cobra v1.8.0 go.mongodb.org/mongo-driver v1.11.6 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sys v0.21.0 - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/sys v0.25.0 + google.golang.org/protobuf v1.34.2 // indirect ) require ( @@ -78,7 +77,7 @@ require ( ) require ( - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -98,8 +97,8 @@ require ( github.com/zmap/zlint/v3 v3.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.26.0 - golang.org/x/text v0.16.0 + golang.org/x/net v0.29.0 + golang.org/x/text v0.18.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect k8s.io/klog/v2 v2.120.1 // indirect ) @@ -112,6 +111,8 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 github.com/emirpasic/gods v1.18.1 github.com/getsentry/sentry-go v0.28.1 + github.com/go-chi/chi/v5 v5.1.0 + github.com/go-chi/render v1.0.3 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 github.com/jackc/pgproto3/v2 v2.3.2 @@ -122,13 +123,26 @@ require ( github.com/xdg-go/scram v1.1.1 github.com/xdg-go/stringprep v1.0.4 github.com/yudai/gojsondiff v1.0.0 - golang.org/x/sync v0.7.0 - golang.org/x/term v0.21.0 + golang.org/x/sync v0.8.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/kustomize/kyaml v0.17.2 ) -require github.com/perimeterx/marshmallow v1.1.5 // indirect +require ( + github.com/ajg/form v1.5.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect +) require ( github.com/alecthomas/chroma v0.10.0 // indirect diff --git a/go.sum b/go.sum index 56768491b0..a8a3e58555 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= @@ -16,6 +18,8 @@ github.com/aymanbagabas/go-osc52 v1.0.3 h1:DTwqENW7X9arYimJrPeGZcV0ln14sGMt3pHZs github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cilium/ebpf v0.13.2 h1:uhLimLX+jF9BTPPvoCUYh/mBeoONkjgaJ9w9fn0mRj4= @@ -34,12 +38,12 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.4+incompatible h1:s/LVDftw9hjblvqIeTiGYXBCD95nOEEl7qRsRrIOuQI= -github.com/docker/docker v24.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= +github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -48,6 +52,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -58,10 +64,17 @@ github.com/getkin/kin-openapi v0.126.0 h1:c2cSgLnAsS0xYfKsgt5oBV6MYRM/giU8/RtwUY github.com/getkin/kin-openapi v0.126.0/go.mod h1:7mONz8IwmSRg6RttPu6v8U/OJ+gr+J99qSFNjPGSQqw= github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= +github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -113,6 +126,9 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -130,8 +146,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/k0kubun/pp/v3 v3.2.0 h1:h33hNTZ9nVFNP3u2Fsgz8JXiF5JINoZfFq4SvKJwNcs= github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapdz1EwA= -github.com/keploy/pgproto3/v2 v2.0.5 h1:8spdNKZ+nOnHVxiimDsqulBRN6viPXPghkA7xppnzJ8= -github.com/keploy/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/keploy/pgproto3/v2 v2.0.7 h1:cBQo5N3zZsQTnQC/c6gLqjrPSSIeao7bInNVLdniyr8= +github.com/keploy/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -172,6 +188,8 @@ github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby v26.0.2+incompatible h1:t41TD3nRvK8E6bZFJdKrmNlH8Xe3epTmdNXf/mnfLKk= github.com/moby/moby v26.0.2+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -330,6 +348,22 @@ github.com/zmap/zlint/v3 v3.1.0 h1:WjVytZo79m/L1+/Mlphl09WBob6YTGljN5IGWZFpAv0= github.com/zmap/zlint/v3 v3.1.0/go.mod h1:L7t8s3sEKkb0A2BxGy1IWrxt1ZATa1R4QfJZaQOD3zU= go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -344,8 +378,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -365,16 +399,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -401,21 +435,21 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -430,14 +464,21 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= +google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index fb28072f00..da7327104d 100755 --- a/main.go +++ b/main.go @@ -14,8 +14,10 @@ import ( userDb "go.keploy.io/server/v2/pkg/platform/yaml/configdb/user" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/utils/log" + //pprof for debugging - //_ "net/http/pprof" + + _ "net/http/pprof" ) // version is the version of the server and will be injected during build by ldflags, same with dsn @@ -30,7 +32,7 @@ func main() { // Uncomment the following code to enable pprof for debugging // go func() { // fmt.Println("Starting pprof server for debugging...") - // err := http.ListenAndServe("localhost:6060", nil) + // err := http.ListenAndServe("0.0.0.0:6060", nil) // if err != nil { // fmt.Println("Failed to start the pprof server for debugging", err) // return diff --git a/pkg/core/core_others.go b/pkg/agent/core_others.go similarity index 98% rename from pkg/core/core_others.go rename to pkg/agent/core_others.go index 8446cde7a6..703626a3c8 100644 --- a/pkg/core/core_others.go +++ b/pkg/agent/core_others.go @@ -1,7 +1,7 @@ //go:build !linux // Package core provides functionality for managing core functionalities in Keploy. -package core +package agent import ( "context" diff --git a/pkg/core/hooks/README.md b/pkg/agent/hooks/README.md similarity index 100% rename from pkg/core/hooks/README.md rename to pkg/agent/hooks/README.md diff --git a/pkg/core/hooks/bpf_arm64_bpfel.go b/pkg/agent/hooks/bpf_arm64_bpfel.go similarity index 98% rename from pkg/core/hooks/bpf_arm64_bpfel.go rename to pkg/agent/hooks/bpf_arm64_bpfel.go index df9cc8dbec..af17c81b4a 100644 --- a/pkg/core/hooks/bpf_arm64_bpfel.go +++ b/pkg/agent/hooks/bpf_arm64_bpfel.go @@ -95,11 +95,11 @@ type bpfMapSpecs struct { ConnInfoMap *ebpf.MapSpec `ebpf:"conn_info_map"` CurrentSockMap *ebpf.MapSpec `ebpf:"current_sock_map"` DestInfoMap *ebpf.MapSpec `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.MapSpec `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.MapSpec `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.MapSpec `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.MapSpec `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.MapSpec `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.MapSpec `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.MapSpec `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.MapSpec `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.MapSpec `ebpf:"socket_data_event_buffer_heap"` @@ -135,11 +135,11 @@ type bpfMaps struct { ConnInfoMap *ebpf.Map `ebpf:"conn_info_map"` CurrentSockMap *ebpf.Map `ebpf:"current_sock_map"` DestInfoMap *ebpf.Map `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.Map `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.Map `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.Map `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.Map `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.Map `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.Map `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.Map `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.Map `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.Map `ebpf:"socket_data_event_buffer_heap"` @@ -158,11 +158,11 @@ func (m *bpfMaps) Close() error { m.ConnInfoMap, m.CurrentSockMap, m.DestInfoMap, - m.DockerAppRegistrationMap, m.KeployAgentKernelPidMap, m.KeployAgentRegistrationMap, m.KeployClientKernelPidMap, m.KeployClientRegistrationMap, + m.KeployProxyInfo, m.RedirectProxyMap, m.SocketCloseEvents, m.SocketDataEventBufferHeap, diff --git a/pkg/agent/hooks/bpf_arm64_bpfel.o b/pkg/agent/hooks/bpf_arm64_bpfel.o new file mode 100644 index 0000000000..b87829d8c7 Binary files /dev/null and b/pkg/agent/hooks/bpf_arm64_bpfel.o differ diff --git a/pkg/core/hooks/bpf_x86_bpfel.go b/pkg/agent/hooks/bpf_x86_bpfel.go similarity index 98% rename from pkg/core/hooks/bpf_x86_bpfel.go rename to pkg/agent/hooks/bpf_x86_bpfel.go index cf8f135116..99ef0483c7 100644 --- a/pkg/core/hooks/bpf_x86_bpfel.go +++ b/pkg/agent/hooks/bpf_x86_bpfel.go @@ -95,11 +95,11 @@ type bpfMapSpecs struct { ConnInfoMap *ebpf.MapSpec `ebpf:"conn_info_map"` CurrentSockMap *ebpf.MapSpec `ebpf:"current_sock_map"` DestInfoMap *ebpf.MapSpec `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.MapSpec `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.MapSpec `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.MapSpec `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.MapSpec `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.MapSpec `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.MapSpec `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.MapSpec `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.MapSpec `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.MapSpec `ebpf:"socket_data_event_buffer_heap"` @@ -135,11 +135,11 @@ type bpfMaps struct { ConnInfoMap *ebpf.Map `ebpf:"conn_info_map"` CurrentSockMap *ebpf.Map `ebpf:"current_sock_map"` DestInfoMap *ebpf.Map `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.Map `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.Map `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.Map `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.Map `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.Map `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.Map `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.Map `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.Map `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.Map `ebpf:"socket_data_event_buffer_heap"` @@ -158,11 +158,11 @@ func (m *bpfMaps) Close() error { m.ConnInfoMap, m.CurrentSockMap, m.DestInfoMap, - m.DockerAppRegistrationMap, m.KeployAgentKernelPidMap, m.KeployAgentRegistrationMap, m.KeployClientKernelPidMap, m.KeployClientRegistrationMap, + m.KeployProxyInfo, m.RedirectProxyMap, m.SocketCloseEvents, m.SocketDataEventBufferHeap, diff --git a/pkg/agent/hooks/bpf_x86_bpfel.o b/pkg/agent/hooks/bpf_x86_bpfel.o new file mode 100644 index 0000000000..2b6b90d0c2 Binary files /dev/null and b/pkg/agent/hooks/bpf_x86_bpfel.o differ diff --git a/pkg/core/hooks/conn/README.md b/pkg/agent/hooks/conn/README.md similarity index 100% rename from pkg/core/hooks/conn/README.md rename to pkg/agent/hooks/conn/README.md diff --git a/pkg/core/hooks/conn/conn.go b/pkg/agent/hooks/conn/conn.go similarity index 100% rename from pkg/core/hooks/conn/conn.go rename to pkg/agent/hooks/conn/conn.go diff --git a/pkg/core/hooks/conn/factory.go b/pkg/agent/hooks/conn/factory.go similarity index 87% rename from pkg/core/hooks/conn/factory.go rename to pkg/agent/hooks/conn/factory.go index 1156922814..0f9d4ec479 100755 --- a/pkg/core/hooks/conn/factory.go +++ b/pkg/agent/hooks/conn/factory.go @@ -43,7 +43,7 @@ func NewFactory(inactivityThreshold time.Duration, logger *zap.Logger) *Factory // ProcessActiveTrackers iterates over all conn the trackers and checks if they are complete. If so, it captures the ingress call and // deletes the tracker. If the tracker is inactive for a long time, it deletes it. -func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *models.TestCase, opts models.IncomingOptions) { +func (factory *Factory) ProcessActiveTrackers(ctx context.Context, testMap *sync.Map, opts models.IncomingOptions) { factory.mutex.Lock() defer factory.mutex.Unlock() var trackersToDelete []ID @@ -52,9 +52,10 @@ func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *model case <-ctx.Done(): return default: - ok, requestBuf, responseBuf, reqTimestampTest, resTimestampTest := tracker.IsComplete() + ok, requestBuf, responseBuf, reqTimestampTest, resTimestampTest, clientID := tracker.IsComplete() if ok { - + fmt.Println("Processing the tracker with key: ", connID) + fmt.Println("Request Buffer::::::::: ", string(requestBuf)) if len(requestBuf) == 0 || len(responseBuf) == 0 { factory.logger.Warn("failed processing a request due to invalid request or response", zap.Any("Request Size", len(requestBuf)), zap.Any("Response Size", len(responseBuf))) continue @@ -70,7 +71,23 @@ func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *model utils.LogError(factory.logger, err, "failed to parse the http response from byte array", zap.Any("responseBuf", responseBuf)) continue } - capture(ctx, factory.logger, t, parsedHTTPReq, parsedHTTPRes, reqTimestampTest, resTimestampTest, opts) + + //get the channel from the test map + // failed to get the channel from the test map, if the client id is not found + t, ok := testMap.Load(clientID) + if !ok { + factory.logger.Error("failed to get the channel from the test map") + continue + } + + // type assert the channel + tc, ok := t.(chan *models.TestCase) + if !ok { + factory.logger.Error("failed to type assert the channel from the test map") + continue + } + + capture(ctx, factory.logger, tc, parsedHTTPReq, parsedHTTPRes, reqTimestampTest, resTimestampTest, opts) } else if tracker.IsInactive(factory.inactivityThreshold) { trackersToDelete = append(trackersToDelete, connID) @@ -80,6 +97,7 @@ func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *model // Delete all the processed trackers. for _, key := range trackersToDelete { + fmt.Println("Deleting the tracker with key: ", key) delete(factory.connections, key) } } @@ -104,6 +122,7 @@ func capture(_ context.Context, logger *zap.Logger, t chan *models.TestCase, req return } + fmt.Println("Request Body::::::: ", string(reqBody)) defer func() { err := resp.Body.Close() if err != nil { diff --git a/pkg/core/hooks/conn/socket.go b/pkg/agent/hooks/conn/socket.go similarity index 79% rename from pkg/core/hooks/conn/socket.go rename to pkg/agent/hooks/conn/socket.go index d7af88596f..58cf799632 100644 --- a/pkg/core/hooks/conn/socket.go +++ b/pkg/agent/hooks/conn/socket.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "sync" "time" "unsafe" @@ -26,18 +27,19 @@ import ( var eventAttributesSize = int(unsafe.Sizeof(SocketDataEvent{})) // ListenSocket starts the socket event listeners -func ListenSocket(ctx context.Context, l *zap.Logger, openMap, dataMap, closeMap *ebpf.Map, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - t := make(chan *models.TestCase, 500) +func ListenSocket(ctx context.Context, l *zap.Logger, clientID uint64, testMap *sync.Map, openMap, dataMap, closeMap *ebpf.Map, opts models.IncomingOptions) error { + err := initRealTimeOffset() if err != nil { utils.LogError(l, err, "failed to initialize real time offset") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } c := NewFactory(time.Minute, l) g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) if !ok { - return nil, errors.New("failed to get the error group from the context") + return errors.New("failed to get the error group from the context") } + fmt.Println("Starting the socket listener", c.connections) g.Go(func() error { defer utils.Recover(l) go func() { @@ -45,35 +47,49 @@ func ListenSocket(ctx context.Context, l *zap.Logger, openMap, dataMap, closeMap for { select { case <-ctx.Done(): + fmt.Println("Context Done in ListenSocket") return default: // TODO refactor this to directly consume the events from the maps - c.ProcessActiveTrackers(ctx, t, opts) + c.ProcessActiveTrackers(ctx, testMap, opts) time.Sleep(100 * time.Millisecond) } } }() <-ctx.Done() - close(t) + + //get the channel from test map and close it + t, ok := testMap.Load(clientID) + if ok { + tc, ok := t.(chan *models.TestCase) + if ok { + // Close the channel when the context is done + close(tc) + } else { + println("Failed to type assert the channel from the test map") + } + } + return nil }) err = open(ctx, c, l, openMap) if err != nil { utils.LogError(l, err, "failed to start open socket listener") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } - err = data(ctx, c, l, dataMap) + + err = data(ctx, c, l, dataMap, clientID) if err != nil { utils.LogError(l, err, "failed to start data socket listener") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } err = exit(ctx, c, l, closeMap) if err != nil { utils.LogError(l, err, "failed to start close socket listener") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } - return t, err + return err } func open(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { @@ -128,7 +144,7 @@ func open(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { return nil } -func data(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { +func data(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map, clientID uint64) error { r, err := ringbuf.NewReader(m) if err != nil { utils.LogError(l, nil, "failed to create ring buffer of socketDataEvent") @@ -176,6 +192,21 @@ func data(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { l.Debug(fmt.Sprintf("Request EntryTimestamp :%v\n", convertUnixNanoToTime(event.EntryTimestampNano))) } + // if event.ClientID == clientID { + // fmt.Println("Event received for the keploy test client") + // continue + // } + + // if event.ClientID != id { + // // log the expected client id and the received client id + // l.Info(fmt.Sprintf("Expected ClientID: %v, Received ClientID: %v", id, event.ClientID)) + + // continue + // } + + fmt.Println("SocketDataEvent-1: ", event.ClientID) + fmt.Printf("Direction: %v\n", event.Direction, "Actual Message: ", string(event.Msg[:event.MsgSize])) + c.GetOrCreate(event.ConnID).AddDataEvent(event) } }() diff --git a/pkg/core/hooks/conn/tracker.go b/pkg/agent/hooks/conn/tracker.go similarity index 96% rename from pkg/core/hooks/conn/tracker.go rename to pkg/agent/hooks/conn/tracker.go index 14bd7e9475..6c2de8d9dd 100755 --- a/pkg/core/hooks/conn/tracker.go +++ b/pkg/agent/hooks/conn/tracker.go @@ -66,6 +66,9 @@ type Tracker struct { reqTimestamps []time.Time isNewRequest bool + + //Client Id's array + clientIDs []uint64 } func NewTracker(connID ID, logger *zap.Logger) *Tracker { @@ -107,7 +110,7 @@ func (conn *Tracker) decRecordTestCount() { } // IsComplete checks if the current conn has valid request & response info to capture and also returns the request and response data buffer. -func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { +func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time, uint64) { conn.mutex.Lock() defer conn.mutex.Unlock() @@ -166,7 +169,8 @@ func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { if len(conn.userReqs) > 0 && len(conn.userResps) > 0 { //validated request, response requestBuf = conn.userReqs[0] responseBuf = conn.userResps[0] - + fmt.Println("Request Buffer::::::::: ", string(requestBuf)) + fmt.Println("Response Buffer::::::::: ", string(responseBuf)) //popping out the current request & response data conn.userReqs = conn.userReqs[1:] conn.userResps = conn.userResps[1:] @@ -233,6 +237,7 @@ func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { conn.logger.Debug("unverified recording", zap.Any("recordTraffic", recordTraffic)) } + var clientID uint64 // Checking if record traffic is recorded and request & response timestamp is captured or not. if recordTraffic { if len(conn.reqTimestamps) > 0 { @@ -252,9 +257,16 @@ func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { } conn.logger.Debug(fmt.Sprintf("TestRequestTimestamp:%v || TestResponseTimestamp:%v", reqTimestamps, respTimestamp)) + + //popping out the client id + if len(conn.clientIDs) > 0 { + clientID = conn.clientIDs[0] + conn.clientIDs = conn.clientIDs[1:] + } + } - return recordTraffic, requestBuf, responseBuf, reqTimestamps, respTimestamp + return recordTraffic, requestBuf, responseBuf, reqTimestamps, respTimestamp, clientID } // reset resets the conn's request and response data buffers. @@ -301,6 +313,9 @@ func (conn *Tracker) AddDataEvent(event SocketDataEvent) { // This is to ensure that we capture the response timestamp for the first chunk of the response. if !conn.isNewRequest { conn.isNewRequest = true + + // set the client id + conn.clientIDs = append(conn.clientIDs, event.ClientID) } // Assign the size of the message to the variable msgLengt diff --git a/pkg/core/hooks/conn/util.go b/pkg/agent/hooks/conn/util.go similarity index 98% rename from pkg/core/hooks/conn/util.go rename to pkg/agent/hooks/conn/util.go index a1b80f25f8..e13ff9a968 100755 --- a/pkg/core/hooks/conn/util.go +++ b/pkg/agent/hooks/conn/util.go @@ -10,7 +10,7 @@ import ( "time" "go.keploy.io/server/v2/config" - proxyHttp "go.keploy.io/server/v2/pkg/core/proxy/integrations/http" + proxyHttp "go.keploy.io/server/v2/pkg/agent/proxy/integrations/http" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/hooks/hooks.go b/pkg/agent/hooks/hooks.go similarity index 84% rename from pkg/core/hooks/hooks.go rename to pkg/agent/hooks/hooks.go index 2f589abf69..fbbd879808 100644 --- a/pkg/core/hooks/hooks.go +++ b/pkg/agent/hooks/hooks.go @@ -20,39 +20,44 @@ import ( "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/rlimit" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/hooks/conn" - "go.keploy.io/server/v2/pkg/core/hooks/structs" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks/conn" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" "go.keploy.io/server/v2/pkg/models" "go.uber.org/zap" ) func NewHooks(logger *zap.Logger, cfg *config.Config) *Hooks { + return &Hooks{ logger: logger, - sess: core.NewSessions(), + sess: agent.NewSessions(), m: sync.Mutex{}, proxyIP4: "127.0.0.1", proxyIP6: [4]uint32{0000, 0000, 0000, 0001}, - proxyPort: cfg.ProxyPort, + proxyPort: cfg.Agent.ProxyPort, dnsPort: cfg.DNSPort, + TestMap: &sync.Map{}, + isLoaded: false, } } type Hooks struct { logger *zap.Logger - sess *core.Sessions + sess *agent.Sessions proxyIP4 string proxyIP6 [4]uint32 proxyPort uint32 dnsPort uint32 - - m sync.Mutex + TestMap *sync.Map + m sync.Mutex + isLoaded bool // eBPF C shared maps - clientRegistrationMap *ebpf.Map - agentRegistartionMap *ebpf.Map - dockerAppRegistrationMap *ebpf.Map - redirectProxyMap *ebpf.Map + clientRegistrationMap *ebpf.Map + agentRegistartionMap *ebpf.Map + + redirectProxyMap *ebpf.Map + proxyInfoMap *ebpf.Map //-------------- // eBPF C shared objectsobjects @@ -87,16 +92,16 @@ type Hooks struct { objects bpfObjects writev link.Link writevRet link.Link - appID uint64 } -func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { +func (h *Hooks) Load(ctx context.Context, id uint64, opts agent.HookCfg) error { - h.sess.Set(id, &core.Session{ + fmt.Println("Loading hooks... and setting ID: ", id) + h.sess.Set(id, &agent.Session{ ID: id, }) - err := h.load(ctx, opts) + err := h.load(opts) if err != nil { return err } @@ -119,7 +124,7 @@ func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { return nil } -func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { +func (h *Hooks) load(opts agent.HookCfg) error { // Allow the current process to lock memory for eBPF resources. if err := rlimit.RemoveMemlock(); err != nil { utils.LogError(h.logger, err, "failed to lock memory for eBPF resources") @@ -142,7 +147,8 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.redirectProxyMap = objs.RedirectProxyMap h.clientRegistrationMap = objs.KeployClientRegistrationMap h.agentRegistartionMap = objs.KeployAgentRegistrationMap - h.dockerAppRegistrationMap = objs.DockerAppRegistrationMap + h.proxyInfoMap = objs.KeployProxyInfo + h.objects = objs // --------------- @@ -408,29 +414,6 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.logger.Info("keploy initialized and probes added to the kernel.") - var clientInfo structs.ClientInfo = structs.ClientInfo{} - - switch opts.Mode { - case models.MODE_RECORD: - clientInfo.Mode = uint32(1) - case models.MODE_TEST: - clientInfo.Mode = uint32(2) - default: - clientInfo.Mode = uint32(0) - } - - //sending keploy pid to kernel to get filtered - inode, err := getSelfInodeNumber() - if err != nil { - utils.LogError(h.logger, err, "failed to get inode of the keploy process") - return err - } - - clientInfo.KeployClientInode = inode - clientInfo.KeployClientNsPid = uint32(os.Getpid()) - clientInfo.IsKeployClientRegistered = uint32(0) - h.logger.Debug("Keploy Pid sent successfully...") - if opts.IsDocker { h.proxyIP4 = opts.KeployIPV4 ipv6, err := ToIPv4MappedIPv6(opts.KeployIPV4) @@ -443,55 +426,103 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.logger.Debug("proxy ips", zap.String("ipv4", h.proxyIP4), zap.Any("ipv6", h.proxyIP6)) - proxyIP, err := IPv4ToUint32(h.proxyIP4) + var agentInfo structs.AgentInfo = structs.AgentInfo{} + agentInfo.KeployAgentNsPid = uint32(os.Getpid()) + agentInfo.KeployAgentInode, err = GetSelfInodeNumber() if err != nil { - return fmt.Errorf("failed to convert ip string:[%v] to 32-bit integer", opts.KeployIPV4) + utils.LogError(h.logger, err, "failed to get inode of the keploy process") + return err } - var agentInfo structs.AgentInfo = structs.AgentInfo{} + agentInfo.DNSPort = int32(h.dnsPort) - agentInfo.ProxyInfo = structs.ProxyInfo{ - IP4: proxyIP, - IP6: h.proxyIP6, - Port: h.proxyPort, + // if opts.IsDocker { + // clientInfo.IsDockerApp = uint32(1) + // } else { + // clientInfo.IsDockerApp = uint32(0) + // } + + // ports := GetPortToSendToKernel(ctx, opts.Rules) + // for i := 0; i < 10; i++ { + // if len(ports) <= i { + // clientInfo.PassThroughPorts[i] = -1 + // continue + // } + // clientInfo.PassThroughPorts[i] = int32(ports[i]) + // } + + // for sending client pid to kernel + // fmt.Println("Sending client info to kernel...", clientInfo) + // err = h.SendClientInfo(opts.AppID, clientInfo) + // if err != nil { + // h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + // return err + // } + err = h.SendAgentInfo(agentInfo) + if err != nil { + h.logger.Error("failed to send agent info to the ebpf program", zap.Error(err)) + return err } - agentInfo.DNSPort = int32(h.dnsPort) + return nil +} - if opts.IsDocker { - clientInfo.IsDockerApp = uint32(1) +func (h *Hooks) Record(ctx context.Context, clientID uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + tc := make(chan *models.TestCase, 1) + // create a sync map with key clientId and t as value + // this map will be used to store the test cases for each client + ctx = context.WithoutCancel(ctx) + h.TestMap.Store(clientID, tc) + if !h.isLoaded { + err := conn.ListenSocket(ctx, h.logger, clientID, h.TestMap, h.objects.SocketOpenEvents, h.objects.SocketDataEvents, h.objects.SocketCloseEvents, opts) + if err != nil { + return nil, err + } + h.isLoaded = true } else { - clientInfo.IsDockerApp = uint32(0) - } - - ports := GetPortToSendToKernel(ctx, opts.Rules) - for i := 0; i < 10; i++ { - if len(ports) <= i { - clientInfo.PassThroughPorts[i] = -1 - continue + // read from the map + t, ok := h.TestMap.Load(clientID) + if ok { + tc, ok = t.(chan *models.TestCase) + if ok { + // Close the channel when the context is done + } else { + println("Failed to type assert the channel from the test map") + } } - clientInfo.PassThroughPorts[i] = int32(ports[i]) } - err = h.SendClientInfo(opts.AppID, clientInfo) + // return the receiver of the channel + return tc, nil +} + +func (h *Hooks) SendKeployClientInfo(clientID uint64, clientInfo structs.ClientInfo) error { + + err := h.SendClientInfo(clientID, clientInfo) if err != nil { h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) return err } - err = h.SendAgentInfo(agentInfo) + + return nil +} + +func (h *Hooks) DeleteKeployClientInfo(id uint64) error { + err := h.DeleteClientInfo(id) if err != nil { - h.logger.Error("failed to send agent info to the ebpf program", zap.Error(err)) + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) return err } - return nil } -func (h *Hooks) Record(ctx context.Context, _ uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - // TODO use the session to get the app id - // and then use the app id to get the test cases chan - // and pass that to eBPF consumers/listeners - return conn.ListenSocket(ctx, h.logger, h.objects.SocketOpenEvents, h.objects.SocketDataEvents, h.objects.SocketCloseEvents, opts) +func (h *Hooks) SendClientProxyInfo(clientID uint64, proxyInfo structs.ProxyInfo) error { + err := h.SendProxyInfo(clientID, proxyInfo) + if err != nil { + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + return err + } + return nil } func (h *Hooks) unLoad(_ context.Context) { diff --git a/pkg/core/hooks/kernelComm.go b/pkg/agent/hooks/kernelComm.go similarity index 57% rename from pkg/core/hooks/kernelComm.go rename to pkg/agent/hooks/kernelComm.go index abf8d98d07..fc4eb8ef15 100644 --- a/pkg/core/hooks/kernelComm.go +++ b/pkg/agent/hooks/kernelComm.go @@ -4,33 +4,28 @@ package hooks import ( "context" - "fmt" - - "math/rand" "github.com/cilium/ebpf" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/hooks/structs" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) -//TODO: rename this file. - // Get Used by proxy -func (h *Hooks) Get(_ context.Context, srcPort uint16) (*core.NetworkAddress, error) { +func (h *Hooks) Get(_ context.Context, srcPort uint16) (*agent.NetworkAddress, error) { d, err := h.GetDestinationInfo(srcPort) if err != nil { return nil, err } // TODO : need to implement eBPF code to differentiate between different apps - s, ok := h.sess.Get(0) - if !ok { - return nil, fmt.Errorf("session not found") - } + // s, ok := h.sess.Get(d.ClientID) + // if !ok { + // return nil, fmt.Errorf("session not found") + // } - return &core.NetworkAddress{ - AppID: s.ID, + return &agent.NetworkAddress{ + ClientID: d.ClientID, Version: d.IPVersion, IPv4Addr: d.DestIP4, IPv6Addr: d.DestIP6, @@ -42,7 +37,9 @@ func (h *Hooks) Get(_ context.Context, srcPort uint16) (*core.NetworkAddress, er func (h *Hooks) GetDestinationInfo(srcPort uint16) (*structs.DestInfo, error) { h.m.Lock() defer h.m.Unlock() - destInfo := structs.DestInfo{} + destInfo := structs.DestInfo{ + ClientID: 0, + } if err := h.redirectProxyMap.Lookup(srcPort, &destInfo); err != nil { return nil, err } @@ -65,8 +62,10 @@ func (h *Hooks) CleanProxyEntry(srcPort uint16) error { return nil } -func (h *Hooks) SendClientInfo(id uint64, appInfo structs.ClientInfo) error { - err := h.clientRegistrationMap.Update(id, appInfo, ebpf.UpdateAny) +func (h *Hooks) SendClientInfo(id uint64, clientInfo structs.ClientInfo) error { + h.m.Lock() + defer h.m.Unlock() + err := h.clientRegistrationMap.Update(id, clientInfo, ebpf.UpdateAny) if err != nil { utils.LogError(h.logger, err, "failed to send the app info to the ebpf program") return err @@ -74,30 +73,35 @@ func (h *Hooks) SendClientInfo(id uint64, appInfo structs.ClientInfo) error { return nil } -func (h *Hooks) SendAgentInfo(agentInfo structs.AgentInfo) error { - key := 0 - err := h.agentRegistartionMap.Update(uint32(key), agentInfo, ebpf.UpdateAny) +func (h *Hooks) DeleteClientInfo(id uint64) error { + h.m.Lock() + defer h.m.Unlock() + err := h.clientRegistrationMap.Delete(id) if err != nil { - utils.LogError(h.logger, err, "failed to send the agent info to the ebpf program") + utils.LogError(h.logger, err, "failed to send the app info to the ebpf program") return err } + h.logger.Info("successfully removed the client info from the ebpf program with clientId", zap.Any("clientId", id)) return nil } -func (h *Hooks) SendDockerAppInfo(_ uint64, dockerAppInfo structs.DockerAppInfo) error { - if h.appID != 0 { - err := h.dockerAppRegistrationMap.Delete(h.appID) - if err != nil { - utils.LogError(h.logger, err, "failed to remove entry from dockerAppRegistrationMap") - return err - } +// SendProxyInfo sends the network information to the kernel +func (h *Hooks) SendProxyInfo(id uint64, proxInfo structs.ProxyInfo) error { + h.m.Lock() + defer h.m.Unlock() + err := h.proxyInfoMap.Update(id, proxInfo, ebpf.UpdateAny) + if err != nil { + utils.LogError(h.logger, err, "failed to send the proxy info to the ebpf program") + return err } - r := rand.New(rand.NewSource(rand.Int63())) - randomNum := r.Uint64() - h.appID = randomNum - err := h.dockerAppRegistrationMap.Update(h.appID, dockerAppInfo, ebpf.UpdateAny) + return nil +} + +func (h *Hooks) SendAgentInfo(agentInfo structs.AgentInfo) error { + key := 0 + err := h.agentRegistartionMap.Update(uint32(key), agentInfo, ebpf.UpdateAny) if err != nil { - utils.LogError(h.logger, err, "failed to send the dockerAppInfo info to the ebpf program") + utils.LogError(h.logger, err, "failed to send the agent info to the ebpf program") return err } return nil diff --git a/pkg/core/hooks/structs/structs.go b/pkg/agent/hooks/structs/structs.go similarity index 50% rename from pkg/core/hooks/structs/structs.go rename to pkg/agent/hooks/structs/structs.go index 3e9483d203..1546f2be79 100755 --- a/pkg/core/hooks/structs/structs.go +++ b/pkg/agent/hooks/structs/structs.go @@ -5,15 +5,6 @@ package structs type BpfSpinLock struct{ Val uint32 } -// struct dest_info_t -// { -// u32 ip_version; -// u32 dest_ip4; -// u32 dest_ip6[4]; -// u32 dest_port; -// u32 kernelPid; -// }; - type DestInfo struct { IPVersion uint32 DestIP4 uint32 @@ -23,13 +14,6 @@ type DestInfo struct { ClientID uint64 } -// struct proxy_info -// { -// u32 ip4; -// u32 ip6[4]; -// u32 port; -// }; - type ProxyInfo struct { IP4 uint32 IP6 [4]uint32 @@ -41,17 +25,6 @@ type DockerAppInfo struct { ClientID uint64 } -// struct app_info -// { -// u32 keploy_client_ns_pid; -// u64 keploy_client_inode; -// u64 app_inode; -// u32 mode; -// u32 is_docker_app; -// u32 is_keploy_client_registered; // whether the client is registered or not -// s32 pass_through_ports[PASS_THROUGH_ARRAY_SIZE]; -// }; - type ClientInfo struct { KeployClientInode uint64 KeployClientNsPid uint32 @@ -59,19 +32,11 @@ type ClientInfo struct { IsDockerApp uint32 IsKeployClientRegistered uint32 PassThroughPorts [10]int32 + AppInode uint64 } -// struct agent_info -// { -// u32 keploy_agent_ns_pid; -// u32 keploy_agent_inode; -// struct proxy_info proxy_info; -// s32 dns_port; -// }; - type AgentInfo struct { KeployAgentNsPid uint32 DNSPort int32 KeployAgentInode uint64 - ProxyInfo ProxyInfo } diff --git a/pkg/core/hooks/util.go b/pkg/agent/hooks/util.go similarity index 96% rename from pkg/core/hooks/util.go rename to pkg/agent/hooks/util.go index f28a9ff485..8a33f8e002 100644 --- a/pkg/core/hooks/util.go +++ b/pkg/agent/hooks/util.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package hooks @@ -7,6 +7,7 @@ import ( "context" "encoding/binary" "errors" + "fmt" "net" "os" "path/filepath" @@ -28,6 +29,7 @@ func IPv4ToUint32(ipStr string) (uint32, error) { } return 0, errors.New("not a valid IPv4 address") } + fmt.Println("failed to parse IP address", ipStr) return 0, errors.New("failed to parse IP address") } @@ -97,7 +99,7 @@ func detectCgroupPath(logger *zap.Logger) (string, error) { return "", errors.New("cgroup2 not mounted") } -func getSelfInodeNumber() (uint64, error) { +func GetSelfInodeNumber() (uint64, error) { p := filepath.Join("/proc", "self", "ns", "pid") f, err := os.Stat(p) diff --git a/pkg/core/proxy/README.md b/pkg/agent/proxy/README.md similarity index 100% rename from pkg/core/proxy/README.md rename to pkg/agent/proxy/README.md diff --git a/pkg/core/proxy/dns.go b/pkg/agent/proxy/dns.go similarity index 100% rename from pkg/core/proxy/dns.go rename to pkg/agent/proxy/dns.go diff --git a/pkg/core/proxy/integrations/README.md b/pkg/agent/proxy/integrations/README.md similarity index 100% rename from pkg/core/proxy/integrations/README.md rename to pkg/agent/proxy/integrations/README.md diff --git a/pkg/core/proxy/integrations/generic/README.md b/pkg/agent/proxy/integrations/generic/README.md similarity index 100% rename from pkg/core/proxy/integrations/generic/README.md rename to pkg/agent/proxy/integrations/generic/README.md diff --git a/pkg/core/proxy/integrations/generic/decode.go b/pkg/agent/proxy/integrations/generic/decode.go similarity index 95% rename from pkg/core/proxy/integrations/generic/decode.go rename to pkg/agent/proxy/integrations/generic/decode.go index d7b0af8c84..13708e6263 100644 --- a/pkg/core/proxy/integrations/generic/decode.go +++ b/pkg/agent/proxy/integrations/generic/decode.go @@ -9,9 +9,9 @@ import ( "net" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/generic/encode.go b/pkg/agent/proxy/integrations/generic/encode.go similarity index 98% rename from pkg/core/proxy/integrations/generic/encode.go rename to pkg/agent/proxy/integrations/generic/encode.go index 14e85c49d3..7f880a810a 100644 --- a/pkg/core/proxy/integrations/generic/encode.go +++ b/pkg/agent/proxy/integrations/generic/encode.go @@ -11,8 +11,8 @@ import ( "strconv" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/generic/generic.go b/pkg/agent/proxy/integrations/generic/generic.go similarity index 90% rename from pkg/core/proxy/integrations/generic/generic.go rename to pkg/agent/proxy/integrations/generic/generic.go index 02f4759af9..82f97eb015 100755 --- a/pkg/core/proxy/integrations/generic/generic.go +++ b/pkg/agent/proxy/integrations/generic/generic.go @@ -6,8 +6,8 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -32,7 +32,7 @@ func (g *Generic) MatchType(_ context.Context, _ []byte) bool { return false } -func (g *Generic) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (g *Generic) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := g.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) diff --git a/pkg/core/proxy/integrations/generic/match.go b/pkg/agent/proxy/integrations/generic/match.go similarity index 97% rename from pkg/core/proxy/integrations/generic/match.go rename to pkg/agent/proxy/integrations/generic/match.go index 6ce7fc7005..6f182044e3 100755 --- a/pkg/core/proxy/integrations/generic/match.go +++ b/pkg/agent/proxy/integrations/generic/match.go @@ -8,9 +8,9 @@ import ( "fmt" "math" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" ) diff --git a/pkg/core/proxy/integrations/grpc/decode.go b/pkg/agent/proxy/integrations/grpc/decode.go similarity index 92% rename from pkg/core/proxy/integrations/grpc/decode.go rename to pkg/agent/proxy/integrations/grpc/decode.go index ffa42761a1..d6635e5f0a 100644 --- a/pkg/core/proxy/integrations/grpc/decode.go +++ b/pkg/agent/proxy/integrations/grpc/decode.go @@ -7,7 +7,7 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/grpc/encode.go b/pkg/agent/proxy/integrations/grpc/encode.go similarity index 97% rename from pkg/core/proxy/integrations/grpc/encode.go rename to pkg/agent/proxy/integrations/grpc/encode.go index 39528758ab..ee69edd7ee 100644 --- a/pkg/core/proxy/integrations/grpc/encode.go +++ b/pkg/agent/proxy/integrations/grpc/encode.go @@ -7,7 +7,7 @@ import ( "io" "net" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/grpc/frame.go b/pkg/agent/proxy/integrations/grpc/frame.go similarity index 100% rename from pkg/core/proxy/integrations/grpc/frame.go rename to pkg/agent/proxy/integrations/grpc/frame.go diff --git a/pkg/core/proxy/integrations/grpc/grpc.go b/pkg/agent/proxy/integrations/grpc/grpc.go similarity index 91% rename from pkg/core/proxy/integrations/grpc/grpc.go rename to pkg/agent/proxy/integrations/grpc/grpc.go index 40ae191510..fb433cebe5 100644 --- a/pkg/core/proxy/integrations/grpc/grpc.go +++ b/pkg/agent/proxy/integrations/grpc/grpc.go @@ -7,8 +7,8 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -35,7 +35,7 @@ func (g *Grpc) MatchType(_ context.Context, reqBuf []byte) bool { return bytes.HasPrefix(reqBuf[:], []byte("PRI * HTTP/2")) } -func (g *Grpc) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (g *Grpc) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := g.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) diff --git a/pkg/core/proxy/integrations/grpc/match.go b/pkg/agent/proxy/integrations/grpc/match.go similarity index 97% rename from pkg/core/proxy/integrations/grpc/match.go rename to pkg/agent/proxy/integrations/grpc/match.go index c76b68878f..33a5309bfd 100644 --- a/pkg/core/proxy/integrations/grpc/match.go +++ b/pkg/agent/proxy/integrations/grpc/match.go @@ -6,7 +6,7 @@ import ( "context" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.uber.org/zap" "go.keploy.io/server/v2/pkg/models" diff --git a/pkg/core/proxy/integrations/grpc/stream.go b/pkg/agent/proxy/integrations/grpc/stream.go similarity index 100% rename from pkg/core/proxy/integrations/grpc/stream.go rename to pkg/agent/proxy/integrations/grpc/stream.go diff --git a/pkg/core/proxy/integrations/grpc/transcoder.go b/pkg/agent/proxy/integrations/grpc/transcoder.go similarity index 99% rename from pkg/core/proxy/integrations/grpc/transcoder.go rename to pkg/agent/proxy/integrations/grpc/transcoder.go index d59d149748..918b206b8e 100644 --- a/pkg/core/proxy/integrations/grpc/transcoder.go +++ b/pkg/agent/proxy/integrations/grpc/transcoder.go @@ -7,7 +7,7 @@ import ( "context" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/grpc/util.go b/pkg/agent/proxy/integrations/grpc/util.go similarity index 100% rename from pkg/core/proxy/integrations/grpc/util.go rename to pkg/agent/proxy/integrations/grpc/util.go diff --git a/pkg/core/proxy/integrations/http/README.md b/pkg/agent/proxy/integrations/http/README.md similarity index 100% rename from pkg/core/proxy/integrations/http/README.md rename to pkg/agent/proxy/integrations/http/README.md diff --git a/pkg/core/proxy/integrations/http/decode.go b/pkg/agent/proxy/integrations/http/decode.go similarity index 98% rename from pkg/core/proxy/integrations/http/decode.go rename to pkg/agent/proxy/integrations/http/decode.go index dabef6c319..93833ebeb6 100644 --- a/pkg/core/proxy/integrations/http/decode.go +++ b/pkg/agent/proxy/integrations/http/decode.go @@ -15,8 +15,8 @@ import ( "strconv" "go.keploy.io/server/v2/pkg" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/http/encode.go b/pkg/agent/proxy/integrations/http/encode.go similarity index 98% rename from pkg/core/proxy/integrations/http/encode.go rename to pkg/agent/proxy/integrations/http/encode.go index f551c11d04..3ff260e8f5 100644 --- a/pkg/core/proxy/integrations/http/encode.go +++ b/pkg/agent/proxy/integrations/http/encode.go @@ -13,8 +13,8 @@ import ( "golang.org/x/sync/errgroup" - "go.keploy.io/server/v2/pkg/core/proxy/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/http/http.go b/pkg/agent/proxy/integrations/http/http.go similarity index 94% rename from pkg/core/proxy/integrations/http/http.go rename to pkg/agent/proxy/integrations/http/http.go index 4521864536..c627f4e6fb 100755 --- a/pkg/core/proxy/integrations/http/http.go +++ b/pkg/agent/proxy/integrations/http/http.go @@ -14,8 +14,8 @@ import ( "strconv" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/pkg" @@ -60,10 +60,10 @@ func (h *HTTP) MatchType(_ context.Context, buf []byte) bool { return isHTTP } -func (h *HTTP) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (h *HTTP) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := h.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) - h.logger.Debug("Recording the outgoing http call in record mode") + h.logger.Info("Recording the outgoing http call in record mode") reqBuf, err := util.ReadInitialBuf(ctx, logger, src) if err != nil { @@ -80,7 +80,7 @@ func (h *HTTP) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, m func (h *HTTP) MockOutgoing(ctx context.Context, src net.Conn, dstCfg *models.ConditionalDstCfg, mockDb integrations.MockMemDb, opts models.OutgoingOptions) error { logger := h.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) - h.logger.Debug("Mocking the outgoing http call in test mode") + h.logger.Info("Mocking the outgoing http call in test mode") reqBuf, err := util.ReadInitialBuf(ctx, logger, src) if err != nil { diff --git a/pkg/core/proxy/integrations/http/match.go b/pkg/agent/proxy/integrations/http/match.go similarity index 98% rename from pkg/core/proxy/integrations/http/match.go rename to pkg/agent/proxy/integrations/http/match.go index 48d62e7e2e..0fc5d9af36 100644 --- a/pkg/core/proxy/integrations/http/match.go +++ b/pkg/agent/proxy/integrations/http/match.go @@ -14,8 +14,8 @@ import ( "strings" "github.com/agnivade/levenshtein" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/http/util.go b/pkg/agent/proxy/integrations/http/util.go similarity index 99% rename from pkg/core/proxy/integrations/http/util.go rename to pkg/agent/proxy/integrations/http/util.go index 0de57abf4e..9b2ee03c18 100644 --- a/pkg/core/proxy/integrations/http/util.go +++ b/pkg/agent/proxy/integrations/http/util.go @@ -16,7 +16,7 @@ import ( "strings" "time" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/integrations.go b/pkg/agent/proxy/integrations/integrations.go similarity index 94% rename from pkg/core/proxy/integrations/integrations.go rename to pkg/agent/proxy/integrations/integrations.go index eddffb33f4..0e26c3d856 100644 --- a/pkg/core/proxy/integrations/integrations.go +++ b/pkg/agent/proxy/integrations/integrations.go @@ -31,7 +31,7 @@ var Registered = make(map[string]Initializer) type Integrations interface { MatchType(ctx context.Context, reqBuf []byte) bool - RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error + RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error MockOutgoing(ctx context.Context, src net.Conn, dstCfg *models.ConditionalDstCfg, mockDb MockMemDb, opts models.OutgoingOptions) error } diff --git a/pkg/core/proxy/integrations/mongo/README.md b/pkg/agent/proxy/integrations/mongo/README.md similarity index 100% rename from pkg/core/proxy/integrations/mongo/README.md rename to pkg/agent/proxy/integrations/mongo/README.md diff --git a/pkg/core/proxy/integrations/mongo/command.go b/pkg/agent/proxy/integrations/mongo/command.go similarity index 100% rename from pkg/core/proxy/integrations/mongo/command.go rename to pkg/agent/proxy/integrations/mongo/command.go diff --git a/pkg/core/proxy/integrations/mongo/decode.go b/pkg/agent/proxy/integrations/mongo/decode.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/decode.go rename to pkg/agent/proxy/integrations/mongo/decode.go index 4961b7b1df..2819802fe3 100644 --- a/pkg/core/proxy/integrations/mongo/decode.go +++ b/pkg/agent/proxy/integrations/mongo/decode.go @@ -12,8 +12,8 @@ import ( "strconv" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.mongodb.org/mongo-driver/bson" diff --git a/pkg/core/proxy/integrations/mongo/encode.go b/pkg/agent/proxy/integrations/mongo/encode.go similarity index 97% rename from pkg/core/proxy/integrations/mongo/encode.go rename to pkg/agent/proxy/integrations/mongo/encode.go index 963a0fcecb..bb1c5b79e5 100644 --- a/pkg/core/proxy/integrations/mongo/encode.go +++ b/pkg/agent/proxy/integrations/mongo/encode.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sync/errgroup" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -21,7 +21,7 @@ import ( // encodeMongo records the outgoing mongo messages of the client connection, // decodes the wiremessage binary and writes readable string // to the yaml file. -func (m *Mongo) encodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func (m *Mongo) encodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, clientClose chan bool, _ models.OutgoingOptions) error { errCh := make(chan error, 1) @@ -271,6 +271,10 @@ func (m *Mongo) encodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []by }) select { + case <-clientClose: + fmt.Println("client connection is closed from the mongo parser") + mocks <- &models.Mock{} + return ctx.Err() case <-ctx.Done(): return ctx.Err() case err := <-errCh: diff --git a/pkg/core/proxy/integrations/mongo/match.go b/pkg/agent/proxy/integrations/mongo/match.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/match.go rename to pkg/agent/proxy/integrations/mongo/match.go index 6f13ea0f52..177b3c3246 100644 --- a/pkg/core/proxy/integrations/mongo/match.go +++ b/pkg/agent/proxy/integrations/mongo/match.go @@ -8,7 +8,7 @@ import ( "reflect" "strings" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/utils" "go.mongodb.org/mongo-driver/bson" diff --git a/pkg/core/proxy/integrations/mongo/mongo.go b/pkg/agent/proxy/integrations/mongo/mongo.go similarity index 94% rename from pkg/core/proxy/integrations/mongo/mongo.go rename to pkg/agent/proxy/integrations/mongo/mongo.go index 012a1a0726..af6dd4701a 100644 --- a/pkg/core/proxy/integrations/mongo/mongo.go +++ b/pkg/agent/proxy/integrations/mongo/mongo.go @@ -9,10 +9,10 @@ import ( "sync" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/utils" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.mongodb.org/mongo-driver/x/mongo/driver/wiremessage" "go.uber.org/zap" @@ -48,7 +48,7 @@ func (m *Mongo) MatchType(_ context.Context, buffer []byte) bool { // RecordOutgoing records the outgoing mongo messages of the client connection into the yaml file. // The database connection is keep-alive so, this function will be called during the connection establishment. -func (m *Mongo) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (m *Mongo) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := m.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) if err != nil { @@ -61,7 +61,7 @@ func (m *Mongo) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, // initially the reqBuf contains the first network packet // from the client connection which is used to determine // the packet type in MatchType. - err = m.encodeMongo(ctx, logger, reqBuf, src, dst, mocks, opts) + err = m.encodeMongo(ctx, logger, reqBuf, src, dst, mocks, clientClose, opts) if err != nil { utils.LogError(logger, err, "failed to encode the mongo message into the yaml") return err @@ -82,6 +82,7 @@ func (m *Mongo) MockOutgoing(ctx context.Context, src net.Conn, dstCfg *models.C return err } + m.logger.Info("Mocking the mongo message") // converts the yaml string into the binary packet err = decodeMongo(ctx, logger, reqBuf, src, dstCfg, mockDb, opts) if err != nil { diff --git a/pkg/core/proxy/integrations/mongo/operation.go b/pkg/agent/proxy/integrations/mongo/operation.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/operation.go rename to pkg/agent/proxy/integrations/mongo/operation.go index 9a2fcf9315..98aefde79b 100644 --- a/pkg/core/proxy/integrations/mongo/operation.go +++ b/pkg/agent/proxy/integrations/mongo/operation.go @@ -12,8 +12,8 @@ import ( "strings" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/scram" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/scram" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.mongodb.org/mongo-driver/bson" diff --git a/pkg/core/proxy/integrations/mongo/scramAuth.go b/pkg/agent/proxy/integrations/mongo/scramAuth.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/scramAuth.go rename to pkg/agent/proxy/integrations/mongo/scramAuth.go index e75971ce5a..2b90eb56ae 100644 --- a/pkg/core/proxy/integrations/mongo/scramAuth.go +++ b/pkg/agent/proxy/integrations/mongo/scramAuth.go @@ -11,10 +11,10 @@ import ( "strings" "sync" - scramUtil "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + scramUtil "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/scram" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/scram" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mongo/util.go b/pkg/agent/proxy/integrations/mongo/util.go similarity index 100% rename from pkg/core/proxy/integrations/mongo/util.go rename to pkg/agent/proxy/integrations/mongo/util.go diff --git a/pkg/core/proxy/integrations/mysql/README.md b/pkg/agent/proxy/integrations/mysql/README.md similarity index 100% rename from pkg/core/proxy/integrations/mysql/README.md rename to pkg/agent/proxy/integrations/mysql/README.md diff --git a/pkg/core/proxy/integrations/mysql/mysql.go b/pkg/agent/proxy/integrations/mysql/mysql.go similarity index 85% rename from pkg/core/proxy/integrations/mysql/mysql.go rename to pkg/agent/proxy/integrations/mysql/mysql.go index 1914a5262e..10b7ea9766 100644 --- a/pkg/core/proxy/integrations/mysql/mysql.go +++ b/pkg/agent/proxy/integrations/mysql/mysql.go @@ -8,9 +8,9 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/recorder" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/replayer" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/recorder" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/replayer" "go.keploy.io/server/v2/utils" @@ -37,7 +37,7 @@ func (m *MySQL) MatchType(_ context.Context, _ []byte) bool { return false } -func (m *MySQL) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (m *MySQL) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := m.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) err := recorder.Record(ctx, logger, src, dst, mocks, opts) diff --git a/pkg/core/proxy/integrations/mysql/recorder/conn.go b/pkg/agent/proxy/integrations/mysql/recorder/conn.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/recorder/conn.go rename to pkg/agent/proxy/integrations/mysql/recorder/conn.go index 89ce66ac34..4a21045ee9 100644 --- a/pkg/core/proxy/integrations/mysql/recorder/conn.go +++ b/pkg/agent/proxy/integrations/mysql/recorder/conn.go @@ -11,11 +11,11 @@ import ( "net" "time" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - intgUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pTls "go.keploy.io/server/v2/pkg/core/proxy/tls" - pUtils "go.keploy.io/server/v2/pkg/core/proxy/util" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + intgUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pTls "go.keploy.io/server/v2/pkg/agent/proxy/tls" + pUtils "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/recorder/query.go b/pkg/agent/proxy/integrations/mysql/recorder/query.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/recorder/query.go rename to pkg/agent/proxy/integrations/mysql/recorder/query.go index 122b57ebc4..9edb22b885 100644 --- a/pkg/core/proxy/integrations/mysql/recorder/query.go +++ b/pkg/agent/proxy/integrations/mysql/recorder/query.go @@ -9,9 +9,9 @@ import ( "net" "time" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/recorder/record.go b/pkg/agent/proxy/integrations/mysql/recorder/record.go similarity index 96% rename from pkg/core/proxy/integrations/mysql/recorder/record.go rename to pkg/agent/proxy/integrations/mysql/recorder/record.go index 3b48d234ac..35d5598f6f 100644 --- a/pkg/core/proxy/integrations/mysql/recorder/record.go +++ b/pkg/agent/proxy/integrations/mysql/recorder/record.go @@ -12,8 +12,8 @@ import ( "golang.org/x/sync/errgroup" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/conn.go b/pkg/agent/proxy/integrations/mysql/replayer/conn.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/replayer/conn.go rename to pkg/agent/proxy/integrations/mysql/replayer/conn.go index ea7d09ef50..269bf73186 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/conn.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/conn.go @@ -9,11 +9,11 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - pTls "go.keploy.io/server/v2/pkg/core/proxy/tls" - pUtils "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + pTls "go.keploy.io/server/v2/pkg/agent/proxy/tls" + pUtils "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/match.go b/pkg/agent/proxy/integrations/mysql/replayer/match.go similarity index 99% rename from pkg/core/proxy/integrations/mysql/replayer/match.go rename to pkg/agent/proxy/integrations/mysql/replayer/match.go index 3cc2a335c6..ad5681a298 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/match.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/match.go @@ -9,9 +9,9 @@ import ( "io" "math" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - intgUtil "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + intgUtil "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/query.go b/pkg/agent/proxy/integrations/mysql/replayer/query.go similarity index 94% rename from pkg/core/proxy/integrations/mysql/replayer/query.go rename to pkg/agent/proxy/integrations/mysql/replayer/query.go index 4268b968ec..4a03e8494b 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/query.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/query.go @@ -9,9 +9,9 @@ import ( "net" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/replay.go b/pkg/agent/proxy/integrations/mysql/replayer/replay.go similarity index 92% rename from pkg/core/proxy/integrations/mysql/replayer/replay.go rename to pkg/agent/proxy/integrations/mysql/replayer/replay.go index 106b0f3812..e66d9e1bea 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/replay.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/replay.go @@ -8,10 +8,10 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - intgUtil "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + intgUtil "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/utils/util.go b/pkg/agent/proxy/integrations/mysql/utils/util.go similarity index 99% rename from pkg/core/proxy/integrations/mysql/utils/util.go rename to pkg/agent/proxy/integrations/mysql/utils/util.go index f241a4bcca..a3e8e19739 100644 --- a/pkg/core/proxy/integrations/mysql/utils/util.go +++ b/pkg/agent/proxy/integrations/mysql/utils/util.go @@ -11,7 +11,7 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/mysql/wire/decode.go b/pkg/agent/proxy/integrations/mysql/wire/decode.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/decode.go rename to pkg/agent/proxy/integrations/mysql/wire/decode.go index 40306b515a..8a5c2b3b4c 100644 --- a/pkg/core/proxy/integrations/mysql/wire/decode.go +++ b/pkg/agent/proxy/integrations/mysql/wire/decode.go @@ -8,14 +8,14 @@ import ( "fmt" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase" - connection "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/conn" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/utility" - - itgUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase" + connection "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/conn" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/utility" + + itgUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/mysql/wire/encode.go b/pkg/agent/proxy/integrations/mysql/wire/encode.go similarity index 93% rename from pkg/core/proxy/integrations/mysql/wire/encode.go rename to pkg/agent/proxy/integrations/mysql/wire/encode.go index 566745cc13..286a5dc39b 100644 --- a/pkg/core/proxy/integrations/mysql/wire/encode.go +++ b/pkg/agent/proxy/integrations/mysql/wire/encode.go @@ -8,10 +8,10 @@ import ( "fmt" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/conn" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/conn" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go similarity index 95% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go index ddeb0652b5..5950cd48ed 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go @@ -8,7 +8,7 @@ import ( "errors" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go index 73a8c15a58..93226b9857 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go @@ -9,7 +9,8 @@ import ( "errors" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) @@ -84,7 +85,7 @@ func DecodeHandshakeResponse(_ context.Context, logger *zap.Logger, data []byte) if packet.CapabilityFlags&mysql.CLIENT_CONNECT_WITH_DB != 0 { idx = bytes.IndexByte(data, 0x00) if idx != -1 { - packet.Database = string(data[:idx]) + packet.Database = util.EncodeBase64(data[:idx]) data = data[idx+1:] } } @@ -94,7 +95,7 @@ func DecodeHandshakeResponse(_ context.Context, logger *zap.Logger, data []byte) if idx == -1 { return nil, errors.New("malformed handshake response packet: missing null terminator for AuthPluginName") } - packet.AuthPluginName = string(data[:idx]) + packet.AuthPluginName = util.EncodeBase64(data[:idx]) data = data[idx+1:] } diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/generic.go b/pkg/agent/proxy/integrations/mysql/wire/phase/generic.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/wire/phase/generic.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/generic.go index d0663b76fe..e7d9fa2027 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/generic.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/generic.go @@ -9,7 +9,7 @@ import ( "encoding/binary" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go similarity index 94% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go index 672b09cc26..ebb11cc76c 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go @@ -7,9 +7,9 @@ import ( "context" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go index b7864663e1..6f9037b967 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go index db53c6576d..02b6e43510 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go @@ -8,7 +8,7 @@ import ( "fmt" "io" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/queryPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/queryPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/queryPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/queryPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go similarity index 82% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go index a985fc62ac..30141d8c4e 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go @@ -11,7 +11,7 @@ import ( "fmt" "strings" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) @@ -255,14 +255,37 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow columnEntry := row.Values[i] + fmt.Printf("ColumnEntry: %+v\n", columnEntry) + fmt.Printf("columnEntry.Value: %+v\n", columnEntry.Value) + fmt.Printf("Type of Value: %T\n", columnEntry.Value) + switch columnEntry.Type { case mysql.FieldTypeLong: var val any if columnEntry.Unsigned { - val = uint32(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint32(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint32(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint32(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint32(columnEntry.Value.(uint)) + } } else { - val = int32(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int32(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int32(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int32(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int32(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } @@ -277,10 +300,29 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow case mysql.FieldTypeTiny: var val any if columnEntry.Unsigned { - val = uint8(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint8(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint8(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint8(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint8(columnEntry.Value.(uint)) + } } else { - val = int8(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int8(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int8(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int8(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int8(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } @@ -288,21 +330,60 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow case mysql.FieldTypeShort, mysql.FieldTypeYear: var val any if columnEntry.Unsigned { - val = uint16(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint16(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint16(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint16(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint16(columnEntry.Value.(uint)) + } + } else { - val = int16(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int16(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int16(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int16(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int16(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write int16 value: %w", err) } case mysql.FieldTypeLongLong: var val any if columnEntry.Unsigned { - val = uint64(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint64(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint64(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint64(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint64(columnEntry.Value.(uint)) + } } else { - val = int64(columnEntry.Value.(int)) - } + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int64(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int64(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int64(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int64(columnEntry.Value.(uint)) + } + } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go similarity index 89% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go index 3d16dbefa5..a3da90dde0 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go @@ -6,7 +6,7 @@ import ( "context" "errors" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go index db554015b0..65eddcd317 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go @@ -8,7 +8,7 @@ import ( "encoding/binary" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go index 6c50ebfdc1..294959d0da 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go @@ -8,7 +8,7 @@ import ( "fmt" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/util.go b/pkg/agent/proxy/integrations/mysql/wire/util.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/util.go rename to pkg/agent/proxy/integrations/mysql/wire/util.go diff --git a/pkg/core/proxy/integrations/postgres/v1/decode.go b/pkg/agent/proxy/integrations/postgres/v1/decode.go similarity index 95% rename from pkg/core/proxy/integrations/postgres/v1/decode.go rename to pkg/agent/proxy/integrations/postgres/v1/decode.go index 308f2625bb..b49e171987 100644 --- a/pkg/core/proxy/integrations/postgres/v1/decode.go +++ b/pkg/agent/proxy/integrations/postgres/v1/decode.go @@ -12,9 +12,9 @@ import ( "sync" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -67,7 +67,7 @@ func decodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie } if !matched { - logger.Debug("MISMATCHED REQ is" + string(pgRequests[0])) + logger.Info("MISMATCHED REQ is" + string(pgRequests[0])) _, err = pUtil.PassThrough(ctx, logger, clientConn, dstCfg, pgRequests) if err != nil { utils.LogError(logger, err, "failed to pass the request", zap.Any("request packets", len(pgRequests))) diff --git a/pkg/core/proxy/integrations/postgres/v1/encode.go b/pkg/agent/proxy/integrations/postgres/v1/encode.go similarity index 85% rename from pkg/core/proxy/integrations/postgres/v1/encode.go rename to pkg/agent/proxy/integrations/postgres/v1/encode.go index 90b0d739d6..e506bf9ea7 100755 --- a/pkg/core/proxy/integrations/postgres/v1/encode.go +++ b/pkg/agent/proxy/integrations/postgres/v1/encode.go @@ -12,15 +12,15 @@ import ( "time" "github.com/jackc/pgproto3/v2" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) -func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, clientClose chan bool, _ models.OutgoingOptions) error { logger.Debug("Inside the encodePostgresOutgoing function") var pgRequests []models.Backend @@ -112,6 +112,28 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie for { select { + case <-clientClose: + logger.Debug("client connection is closed") + if !prevChunkWasReq && len(pgRequests) > 0 && len(pgResponses) > 0 { + metadata := make(map[string]string) + metadata["type"] = "config" + // Save the mock + m := &models.Mock{ + Version: models.GetVersion(), + Name: "mocks", + Kind: models.Postgres, + Spec: models.MockSpec{ + PostgresRequests: pgRequests, + PostgresResponses: pgResponses, + ReqTimestampMock: reqTimestampMock, + ResTimestampMock: resTimestampMock, + Metadata: metadata, + }, + ConnectionID: ctx.Value(models.ClientConnectionIDKey).(string), + } + mocks <- m + return ctx.Err() + } case <-ctx.Done(): if !prevChunkWasReq && len(pgRequests) > 0 && len(pgResponses) > 0 { metadata := make(map[string]string) @@ -130,6 +152,7 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie }, ConnectionID: ctx.Value(models.ClientConnectionIDKey).(string), } + fmt.Println("Context is done in the postgres encode function", mocks) return ctx.Err() } case buffer := <-clientBuffChan: @@ -326,10 +349,9 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie // from here take the msg and append its readable form to the pgResponses pgMock := &models.Frontend{ - PacketTypes: pg.FrontendWrapper.PacketTypes, - Identfier: "ServerResponse", - Length: uint32(len(reqBuf)), - // Payload: bufStr, + PacketTypes: pg.FrontendWrapper.PacketTypes, + Identfier: "ServerResponse", + Length: uint32(len(reqBuf)), AuthenticationOk: pg.FrontendWrapper.AuthenticationOk, AuthenticationCleartextPassword: pg.FrontendWrapper.AuthenticationCleartextPassword, AuthenticationMD5Password: pg.FrontendWrapper.AuthenticationMD5Password, @@ -345,24 +367,24 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie CommandCompletes: pg.FrontendWrapper.CommandCompletes, CopyData: pg.FrontendWrapper.CopyData, CopyDone: pg.FrontendWrapper.CopyDone, - CopyInResponse: pg.FrontendWrapper.CopyInResponse, - CopyOutResponse: pg.FrontendWrapper.CopyOutResponse, - DataRow: pg.FrontendWrapper.DataRow, - DataRows: pg.FrontendWrapper.DataRows, - EmptyQueryResponse: pg.FrontendWrapper.EmptyQueryResponse, - ErrorResponse: pg.FrontendWrapper.ErrorResponse, - FunctionCallResponse: pg.FrontendWrapper.FunctionCallResponse, - NoData: pg.FrontendWrapper.NoData, - NoticeResponse: pg.FrontendWrapper.NoticeResponse, - NotificationResponse: pg.FrontendWrapper.NotificationResponse, - ParameterDescription: pg.FrontendWrapper.ParameterDescription, - ParameterStatusCombined: pg.FrontendWrapper.ParameterStatusCombined, - ParseComplete: pg.FrontendWrapper.ParseComplete, - PortalSuspended: pg.FrontendWrapper.PortalSuspended, - ReadyForQuery: pg.FrontendWrapper.ReadyForQuery, - RowDescription: pg.FrontendWrapper.RowDescription, - MsgType: pg.FrontendWrapper.MsgType, - AuthType: pg.FrontendWrapper.AuthType, + // CopyInResponse: pg.FrontendWrapper.CopyInResponse, + // CopyOutResponse: pg.FrontendWrapper.CopyOutResponse, + DataRow: pg.FrontendWrapper.DataRow, + DataRows: pg.FrontendWrapper.DataRows, + EmptyQueryResponse: pg.FrontendWrapper.EmptyQueryResponse, + ErrorResponse: pg.FrontendWrapper.ErrorResponse, + FunctionCallResponse: pg.FrontendWrapper.FunctionCallResponse, + NoData: pg.FrontendWrapper.NoData, + NoticeResponse: pg.FrontendWrapper.NoticeResponse, + NotificationResponse: pg.FrontendWrapper.NotificationResponse, + ParameterDescription: pg.FrontendWrapper.ParameterDescription, + ParameterStatusCombined: pg.FrontendWrapper.ParameterStatusCombined, + ParseComplete: pg.FrontendWrapper.ParseComplete, + PortalSuspended: pg.FrontendWrapper.PortalSuspended, + ReadyForQuery: pg.FrontendWrapper.ReadyForQuery, + RowDescription: pg.FrontendWrapper.RowDescription, + MsgType: pg.FrontendWrapper.MsgType, + AuthType: pg.FrontendWrapper.AuthType, } afterEncoded, err := postgresDecoderFrontend(*pgMock) diff --git a/pkg/core/proxy/integrations/postgres/v1/match.go b/pkg/agent/proxy/integrations/postgres/v1/match.go similarity index 97% rename from pkg/core/proxy/integrations/postgres/v1/match.go rename to pkg/agent/proxy/integrations/postgres/v1/match.go index 819a9bd35d..b3ec217699 100644 --- a/pkg/core/proxy/integrations/postgres/v1/match.go +++ b/pkg/agent/proxy/integrations/postgres/v1/match.go @@ -14,8 +14,8 @@ import ( "github.com/agnivade/levenshtein" "github.com/jackc/pgproto3/v2" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.uber.org/zap" ) @@ -93,7 +93,6 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut reqGoingOn := decodePgRequest(requestBuffers[0], logger) if reqGoingOn != nil { logger.Debug("PacketTypes", zap.Any("PacketTypes", reqGoingOn.PacketTypes)) - // fmt.Println("REQUEST GOING ON - ", reqGoingOn) logger.Debug("ConnectionId-", zap.String("ConnectionId", ConnectionID)) logger.Debug("TestMap*****", zap.Any("TestMap", testmap)) } @@ -142,7 +141,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut } return true, []models.Frontend{ssl}, nil case initMock.Spec.PostgresRequests[requestIndex].Identfier == "StartupRequest" && isStartupPacket(reqBuff) && initMock.Spec.PostgresRequests[requestIndex].Payload != "AAAACATSFi8=" && initMock.Spec.PostgresResponses[requestIndex].AuthType == 10: - logger.Debug("CHANGING TO MD5 for Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) + logger.Info("CHANGING TO MD5 for Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) res := make([]models.Frontend, len(initMock.Spec.PostgresResponses)) copy(res, initMock.Spec.PostgresResponses) res[requestIndex].AuthType = 5 @@ -223,7 +222,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut getTestPS(requestBuffers, logger, ConnectionID) } - logger.Debug("Sorted Mocks inside pg parser: ", zap.Any("Len of sortedTcsMocks", len(sortedTcsMocks))) + logger.Info("Sorted Mocks inside pg parser: ", zap.Any("Len of sortedTcsMocks", len(sortedTcsMocks))) var matched, sorted bool var idx int @@ -480,7 +479,7 @@ func changeResToPS(mock *models.Mock, actualPgReq *models.Backend, logger *zap.L mockPackets := mock.Spec.PostgresRequests[0].PacketTypes // [P, B, E, P, B, D, E] => [B, E, B, E] - // write code that of packet is ["B", "E"] and mockPackets ["P", "B", "D", "E"] handle it in case1 + // packet is ["B", "E"] and mockPackets ["P", "B", "D", "E"] handle it in case1 // and if packet is [B, E, B, E] and mockPackets [P, B, E, P, B, D, E] handle it in case2 ischanged := false @@ -534,7 +533,6 @@ func changeResToPS(mock *models.Mock, actualPgReq *models.Backend, logger *zap.L break } } - //Matched In Binary Matching for Unsorted mock-222 ischanged2 := false ps2 := actualPgReq.Binds[1].PreparedStatement for _, v := range testmap[connectionID] { @@ -551,7 +549,6 @@ func changeResToPS(mock *models.Mock, actualPgReq *models.Backend, logger *zap.L // Case 4 if reflect.DeepEqual(actualpackets, []string{"B", "E", "B", "E"}) && reflect.DeepEqual(mockPackets, []string{"B", "E", "P", "B", "D", "E"}) { - // logger.Debug("Handling Case 4 for mock", mock.Name) // get the query for the prepared statement of test mode ischanged := false ps := actualPgReq.Binds[1].PreparedStatement diff --git a/pkg/core/proxy/integrations/postgres/v1/postgres.go b/pkg/agent/proxy/integrations/postgres/v1/postgres.go similarity index 89% rename from pkg/core/proxy/integrations/postgres/v1/postgres.go rename to pkg/agent/proxy/integrations/postgres/v1/postgres.go index a4eecaaaaf..1b7ddabdf4 100755 --- a/pkg/core/proxy/integrations/postgres/v1/postgres.go +++ b/pkg/agent/proxy/integrations/postgres/v1/postgres.go @@ -7,8 +7,8 @@ import ( "encoding/binary" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/pkg/models" @@ -49,7 +49,7 @@ func (p *PostgresV1) MatchType(_ context.Context, reqBuf []byte) bool { return version == ProtocolVersion } -func (p *PostgresV1) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (p *PostgresV1) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := p.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) @@ -57,7 +57,7 @@ func (p *PostgresV1) RecordOutgoing(ctx context.Context, src net.Conn, dst net.C utils.LogError(logger, err, "failed to read the initial postgres message") return err } - err = encodePostgres(ctx, logger, reqBuf, src, dst, mocks, opts) + err = encodePostgres(ctx, logger, reqBuf, src, dst, mocks, clientClose, opts) if err != nil { // TODO: why debug log? logger.Debug("failed to encode the postgres message into the yaml") diff --git a/pkg/core/proxy/integrations/postgres/v1/transcoder.go b/pkg/agent/proxy/integrations/postgres/v1/transcoder.go similarity index 97% rename from pkg/core/proxy/integrations/postgres/v1/transcoder.go rename to pkg/agent/proxy/integrations/postgres/v1/transcoder.go index 9e1ce08c9a..3f2d7059fb 100644 --- a/pkg/core/proxy/integrations/postgres/v1/transcoder.go +++ b/pkg/agent/proxy/integrations/postgres/v1/transcoder.go @@ -30,7 +30,6 @@ func NewFrontend() *FrontendWrapper { // PG Response Packet Transcoder func (b *BackendWrapper) translateToReadableBackend(msgBody []byte) (pgproto3.FrontendMessage, error) { - // fmt.Println("msgType", b.BackendWrapper.MsgType) var msg pgproto3.FrontendMessage switch b.BackendWrapper.MsgType { case 'B': @@ -111,9 +110,9 @@ func (f *FrontendWrapper) translateToReadableResponse(logger *zap.Logger, msgBod case 'E': msg = &f.FrontendWrapper.ErrorResponse case 'G': - msg = &f.FrontendWrapper.CopyInResponse - case 'H': - msg = &f.FrontendWrapper.CopyOutResponse + // msg = &f.FrontendWrapper.CopyInResponse + // case 'H': + // msg = &f.FrontendWrapper.CopyOutResponse case 'I': msg = &f.FrontendWrapper.EmptyQueryResponse case 'K': @@ -138,8 +137,6 @@ func (f *FrontendWrapper) translateToReadableResponse(logger *zap.Logger, msgBod msg = &f.FrontendWrapper.RowDescription case 'V': msg = &f.FrontendWrapper.FunctionCallResponse - case 'W': - msg = &f.FrontendWrapper.CopyBothResponse case 'Z': msg = &f.FrontendWrapper.ReadyForQuery default: diff --git a/pkg/core/proxy/integrations/postgres/v1/util.go b/pkg/agent/proxy/integrations/postgres/v1/util.go similarity index 94% rename from pkg/core/proxy/integrations/postgres/v1/util.go rename to pkg/agent/proxy/integrations/postgres/v1/util.go index a9d3faf34d..2b1df9620e 100755 --- a/pkg/core/proxy/integrations/postgres/v1/util.go +++ b/pkg/agent/proxy/integrations/postgres/v1/util.go @@ -10,7 +10,7 @@ import ( "time" "github.com/jackc/pgproto3/v2" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.uber.org/zap" ) @@ -78,16 +78,16 @@ func postgresDecoderFrontend(response models.Frontend) ([]byte, error) { Line: response.ErrorResponse.Line, Routine: response.ErrorResponse.Routine, } - case string('G'): - msg = &pgproto3.CopyInResponse{ - OverallFormat: response.CopyInResponse.OverallFormat, - ColumnFormatCodes: response.CopyInResponse.ColumnFormatCodes, - } - case string('H'): - msg = &pgproto3.CopyOutResponse{ - OverallFormat: response.CopyOutResponse.OverallFormat, - ColumnFormatCodes: response.CopyOutResponse.ColumnFormatCodes, - } + // case string('G'): + // msg = &pgproto3.CopyInResponse{ + // OverallFormat: response.CopyInResponse.OverallFormat, + // ColumnFormatCodes: response.CopyInResponse.ColumnFormatCodes, + // } + // case string('H'): + // msg = &pgproto3.CopyOutResponse{ + // OverallFormat: response.CopyOutResponse.OverallFormat, + // ColumnFormatCodes: response.CopyOutResponse.ColumnFormatCodes, + // } case string('I'): msg = &pgproto3.EmptyQueryResponse{} case string('K'): @@ -165,11 +165,11 @@ func postgresDecoderFrontend(response models.Frontend) ([]byte, error) { msg = &pgproto3.FunctionCallResponse{ Result: response.FunctionCallResponse.Result, } - case string('W'): - msg = &pgproto3.CopyBothResponse{ - OverallFormat: response.CopyBothResponse.OverallFormat, - ColumnFormatCodes: response.CopyBothResponse.ColumnFormatCodes, - } + // case string('W'): + // msg = &pgproto3.CopyBothResponse{ + // // OverallFormat: response.CopyBothResponse.OverallFormat, + // ColumnFormatCodes: response.CopyBothResponse.ColumnFormatCodes, + // } case string('Z'): msg = &pgproto3.ReadyForQuery{ TxStatus: response.ReadyForQuery.TxStatus, @@ -320,7 +320,6 @@ func sliceCommandTag(mock *models.Mock, logger *zap.Logger, prep []QueryData, ac case 1: copyMock := *mock - // fmt.Println("Inside Slice Command Tag for ", psCase) mockPackets := copyMock.Spec.PostgresResponses[0].PacketTypes for idx, v := range mockPackets { if v == "1" { @@ -332,9 +331,7 @@ func sliceCommandTag(mock *models.Mock, logger *zap.Logger, prep []QueryData, ac return ©Mock case 2: - // ["2", D, C, Z] copyMock := *mock - // fmt.Println("Inside Slice Command Tag for ", psCase) mockPackets := copyMock.Spec.PostgresResponses[0].PacketTypes for idx, v := range mockPackets { if v == "1" || v == "T" { @@ -347,11 +344,8 @@ func sliceCommandTag(mock *models.Mock, logger *zap.Logger, prep []QueryData, ac for idx, datarow := range copyMock.Spec.PostgresResponses[0].DataRows { for column, rowVal := range datarow.RowValues { - // fmt.Println("datarow.RowValues", len(datarow.RowValues)) if rsFormat[column] == 1 { - // datarows := make([]byte, 4) newRow, _ := getChandedDataRow(rowVal) - // logger.Info("New Row Value", zap.String("newRow", newRow)) copyMock.Spec.PostgresResponses[0].DataRows[idx].RowValues[column] = newRow } } diff --git a/pkg/core/proxy/integrations/redis/decode.go b/pkg/agent/proxy/integrations/redis/decode.go similarity index 95% rename from pkg/core/proxy/integrations/redis/decode.go rename to pkg/agent/proxy/integrations/redis/decode.go index 4bb83d31b2..6d5a4953f7 100644 --- a/pkg/core/proxy/integrations/redis/decode.go +++ b/pkg/agent/proxy/integrations/redis/decode.go @@ -9,9 +9,9 @@ import ( "net" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/redis/encode.go b/pkg/agent/proxy/integrations/redis/encode.go similarity index 98% rename from pkg/core/proxy/integrations/redis/encode.go rename to pkg/agent/proxy/integrations/redis/encode.go index 7d34d7ce14..658d5335c7 100644 --- a/pkg/core/proxy/integrations/redis/encode.go +++ b/pkg/agent/proxy/integrations/redis/encode.go @@ -11,7 +11,7 @@ import ( "golang.org/x/sync/errgroup" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/redis/match.go b/pkg/agent/proxy/integrations/redis/match.go similarity index 97% rename from pkg/core/proxy/integrations/redis/match.go rename to pkg/agent/proxy/integrations/redis/match.go index 9e3539067c..28debea482 100755 --- a/pkg/core/proxy/integrations/redis/match.go +++ b/pkg/agent/proxy/integrations/redis/match.go @@ -7,9 +7,9 @@ import ( "fmt" "math" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" ) diff --git a/pkg/core/proxy/integrations/redis/redis.go b/pkg/agent/proxy/integrations/redis/redis.go similarity index 91% rename from pkg/core/proxy/integrations/redis/redis.go rename to pkg/agent/proxy/integrations/redis/redis.go index 0a3e7521ca..f4f425e303 100755 --- a/pkg/core/proxy/integrations/redis/redis.go +++ b/pkg/agent/proxy/integrations/redis/redis.go @@ -6,8 +6,8 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -41,7 +41,7 @@ func (r *Redis) MatchType(_ context.Context, buf []byte) bool { } } -func (r *Redis) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (r *Redis) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := r.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) diff --git a/pkg/core/proxy/integrations/scram/scram.go b/pkg/agent/proxy/integrations/scram/scram.go similarity index 99% rename from pkg/core/proxy/integrations/scram/scram.go rename to pkg/agent/proxy/integrations/scram/scram.go index 4b923c565c..8525260d24 100644 --- a/pkg/core/proxy/integrations/scram/scram.go +++ b/pkg/agent/proxy/integrations/scram/scram.go @@ -12,7 +12,7 @@ import ( "github.com/xdg-go/pbkdf2" "github.com/xdg-go/scram" "github.com/xdg-go/stringprep" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/scram/util.go b/pkg/agent/proxy/integrations/scram/util.go similarity index 100% rename from pkg/core/proxy/integrations/scram/util.go rename to pkg/agent/proxy/integrations/scram/util.go diff --git a/pkg/core/proxy/integrations/util/util.go b/pkg/agent/proxy/integrations/util/util.go similarity index 100% rename from pkg/core/proxy/integrations/util/util.go rename to pkg/agent/proxy/integrations/util/util.go diff --git a/pkg/core/proxy/mockmanager.go b/pkg/agent/proxy/mockmanager.go similarity index 100% rename from pkg/core/proxy/mockmanager.go rename to pkg/agent/proxy/mockmanager.go diff --git a/pkg/core/proxy/options.go b/pkg/agent/proxy/options.go similarity index 100% rename from pkg/core/proxy/options.go rename to pkg/agent/proxy/options.go diff --git a/pkg/agent/proxy/parsers.go b/pkg/agent/proxy/parsers.go new file mode 100644 index 0000000000..26ba078f33 --- /dev/null +++ b/pkg/agent/proxy/parsers.go @@ -0,0 +1,14 @@ +//go:build linux + +package proxy + +import ( + // import all the integrations + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/generic" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/grpc" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/http" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mongo" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/postgres/v1" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/redis" +) diff --git a/pkg/core/proxy/proxy.go b/pkg/agent/proxy/proxy.go similarity index 91% rename from pkg/core/proxy/proxy.go rename to pkg/agent/proxy/proxy.go index 2e60be8326..de2c527f93 100755 --- a/pkg/core/proxy/proxy.go +++ b/pkg/agent/proxy/proxy.go @@ -21,11 +21,11 @@ import ( "github.com/miekg/dns" "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" - pTls "go.keploy.io/server/v2/pkg/core/proxy/tls" - "go.keploy.io/server/v2/pkg/core/proxy/util" + pTls "go.keploy.io/server/v2/pkg/agent/proxy/tls" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -39,19 +39,21 @@ type Proxy struct { Port uint32 DNSPort uint32 - DestInfo core.DestInfo + DestInfo agent.DestInfo Integrations map[string]integrations.Integrations MockManagers sync.Map - sessions *core.Sessions + sessions *agent.Sessions connMutex *sync.Mutex ipMutex *sync.Mutex clientConnections []net.Conn - Listener net.Listener + // channel to mark client connection as closed + clientClose chan bool + Listener net.Listener //to store the nsswitch.conf file data nsswitchData []byte // in test mode we change the configuration of "hosts" in nsswitch.conf file to disable resolution over unix socket @@ -59,17 +61,19 @@ type Proxy struct { TCPDNSServer *dns.Server } -func New(logger *zap.Logger, info core.DestInfo, opts *config.Config) *Proxy { +func New(logger *zap.Logger, info agent.DestInfo, opts *config.Config) *Proxy { + return &Proxy{ logger: logger, - Port: opts.ProxyPort, // default: 16789 + Port: opts.Agent.ProxyPort, // default: 16789 DNSPort: opts.DNSPort, // default: 26789 IP4: "127.0.0.1", // default: "127.0.0.1" <-> (2130706433) IP6: "::1", //default: "::1" <-> ([4]uint32{0000, 0000, 0000, 0001}) ipMutex: &sync.Mutex{}, connMutex: &sync.Mutex{}, DestInfo: info, - sessions: core.NewSessions(), + sessions: agent.NewSessions(), // sessions to store the session rules + clientClose: make(chan bool), MockManagers: sync.Map{}, Integrations: make(map[string]integrations.Integrations), } @@ -84,7 +88,7 @@ func (p *Proxy) InitIntegrations(_ context.Context) error { return nil } -func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { +func (p *Proxy) StartProxy(ctx context.Context, opts agent.ProxyOptions) error { //first initialize the integrations err := p.InitIntegrations(ctx) @@ -92,7 +96,6 @@ func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { utils.LogError(p.logger, err, "failed to initialize the integrations") return err } - // set up the CA for tls connections err = pTls.SetupCA(ctx, p.logger) if err != nil { @@ -200,6 +203,7 @@ func (p *Proxy) start(ctx context.Context, readyChan chan<- error) error { readyChan <- err return err } + p.Listener = listener p.logger.Debug(fmt.Sprintf("Proxy server is listening on %s", fmt.Sprintf(":%v", listener.Addr()))) // Signal that the server is ready @@ -219,7 +223,7 @@ func (p *Proxy) start(ctx context.Context, readyChan chan<- error) error { clientConnCancel() err := clientConnErrGrp.Wait() if err != nil { - p.logger.Debug("failed to handle the client connection", zap.Error(err)) + p.logger.Info("failed to handle the client connection", zap.Error(err)) } //closing all the mock channels (if any in record mode) for _, mc := range p.sessions.GetAllMC() { @@ -294,8 +298,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { remoteAddr := srcConn.RemoteAddr().(*net.TCPAddr) sourcePort := remoteAddr.Port - p.logger.Debug("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) - + p.logger.Info("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) destInfo, err := p.DestInfo.Get(ctx, uint16(sourcePort)) if err != nil { utils.LogError(p.logger, err, "failed to fetch the destination info", zap.Any("Source port", sourcePort)) @@ -310,9 +313,10 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { } //get the session rule - rule, ok := p.sessions.Get(destInfo.AppID) + fmt.Println("destInfo.ClientID:::", destInfo.ClientID) + rule, ok := p.sessions.Get(destInfo.ClientID) if !ok { - utils.LogError(p.logger, nil, "failed to fetch the session rule", zap.Any("AppID", destInfo.AppID)) + utils.LogError(p.logger, nil, "failed to fetch the session rule", zap.Any("AppID", destInfo.ClientID)) return err } @@ -391,7 +395,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { rule.OutgoingOptions.DstCfg = dstCfg // Record the outgoing message into a mock - err := p.Integrations["mysql"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, rule.OutgoingOptions) + err := p.Integrations["mysql"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, p.clientClose, rule.OutgoingOptions) if err != nil { utils.LogError(p.logger, err, "failed to record the outgoing message") return err @@ -399,9 +403,9 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { return nil } - m, ok := p.MockManagers.Load(destInfo.AppID) + m, ok := p.MockManagers.Load(destInfo.ClientID) if !ok { - utils.LogError(p.logger, nil, "failed to fetch the mock manager", zap.Any("AppID", destInfo.AppID)) + utils.LogError(p.logger, nil, "failed to fetch the mock manager", zap.Any("AppID", destInfo.ClientID)) return err } @@ -505,9 +509,9 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { } // get the mock manager for the current app - m, ok := p.MockManagers.Load(destInfo.AppID) + m, ok := p.MockManagers.Load(destInfo.ClientID) if !ok { - utils.LogError(logger, err, "failed to fetch the mock manager", zap.Any("AppID", destInfo.AppID)) + utils.LogError(logger, err, "failed to fetch the mock manager", zap.Any("ClientID", destInfo.ClientID)) return err } @@ -517,7 +521,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { for _, parser := range p.Integrations { if parser.MatchType(parserCtx, initialBuf) { if rule.Mode == models.MODE_RECORD { - err := parser.RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, rule.OutgoingOptions) + err := parser.RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, p.clientClose, rule.OutgoingOptions) if err != nil { utils.LogError(logger, err, "failed to record the outgoing message") return err @@ -536,7 +540,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { if generic { logger.Debug("The external dependency is not supported. Hence using generic parser") if rule.Mode == models.MODE_RECORD { - err := p.Integrations["generic"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, rule.OutgoingOptions) + err := p.Integrations["generic"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, p.clientClose, rule.OutgoingOptions) if err != nil { utils.LogError(logger, err, "failed to record the outgoing message") return err @@ -583,14 +587,20 @@ func (p *Proxy) StopProxyServer(ctx context.Context) { p.logger.Info("proxy stopped...") } +func (p *Proxy) MakeClientDeRegisterd(_ context.Context) error { + p.logger.Info("Inside MakeClientDeregisterd of proxyServer") + p.clientClose <- true + return nil +} + func (p *Proxy) Record(_ context.Context, id uint64, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { - p.sessions.Set(id, &core.Session{ + fmt.Println("Inside Record of proxyServer", id) + p.sessions.Set(id, &agent.Session{ ID: id, Mode: models.MODE_RECORD, MC: mocks, OutgoingOptions: opts, }) - p.MockManagers.Store(id, NewMockManager(NewTreeDb(customComparator), NewTreeDb(customComparator), p.logger)) ////set the new proxy ip:port for a new session @@ -603,7 +613,7 @@ func (p *Proxy) Record(_ context.Context, id uint64, mocks chan<- *models.Mock, } func (p *Proxy) Mock(_ context.Context, id uint64, opts models.OutgoingOptions) error { - p.sessions.Set(id, &core.Session{ + p.sessions.Set(id, &agent.Session{ ID: id, Mode: models.MODE_TEST, OutgoingOptions: opts, @@ -642,7 +652,6 @@ func (p *Proxy) SetMocks(_ context.Context, id uint64, filtered []*models.Mock, m.(*MockManager).SetFilteredMocks(filtered) m.(*MockManager).SetUnFilteredMocks(unFiltered) } - return nil } diff --git a/pkg/core/proxy/tls/asset/ca.crt b/pkg/agent/proxy/tls/asset/ca.crt similarity index 100% rename from pkg/core/proxy/tls/asset/ca.crt rename to pkg/agent/proxy/tls/asset/ca.crt diff --git a/pkg/core/proxy/tls/asset/ca.key b/pkg/agent/proxy/tls/asset/ca.key similarity index 100% rename from pkg/core/proxy/tls/asset/ca.key rename to pkg/agent/proxy/tls/asset/ca.key diff --git a/pkg/core/proxy/tls/asset/setup_ca.sh b/pkg/agent/proxy/tls/asset/setup_ca.sh similarity index 100% rename from pkg/core/proxy/tls/asset/setup_ca.sh rename to pkg/agent/proxy/tls/asset/setup_ca.sh diff --git a/pkg/core/proxy/tls/ca.go b/pkg/agent/proxy/tls/ca.go similarity index 99% rename from pkg/core/proxy/tls/ca.go rename to pkg/agent/proxy/tls/ca.go index 0bd4e68e5a..d0189d1222 100644 --- a/pkg/core/proxy/tls/ca.go +++ b/pkg/agent/proxy/tls/ca.go @@ -18,7 +18,7 @@ import ( cfsslLog "github.com/cloudflare/cfssl/log" "github.com/cloudflare/cfssl/signer" "github.com/cloudflare/cfssl/signer/local" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/tls/tls.go b/pkg/agent/proxy/tls/tls.go similarity index 100% rename from pkg/core/proxy/tls/tls.go rename to pkg/agent/proxy/tls/tls.go diff --git a/pkg/core/proxy/treedb.go b/pkg/agent/proxy/treedb.go similarity index 100% rename from pkg/core/proxy/treedb.go rename to pkg/agent/proxy/treedb.go diff --git a/pkg/core/proxy/util.go b/pkg/agent/proxy/util.go similarity index 97% rename from pkg/core/proxy/util.go rename to pkg/agent/proxy/util.go index 2545148482..5db09b76ef 100644 --- a/pkg/core/proxy/util.go +++ b/pkg/agent/proxy/util.go @@ -9,7 +9,7 @@ import ( "net" "os" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/util/util.go b/pkg/agent/proxy/util/util.go similarity index 100% rename from pkg/core/proxy/util/util.go rename to pkg/agent/proxy/util/util.go diff --git a/pkg/agent/routes/record.go b/pkg/agent/routes/record.go new file mode 100644 index 0000000000..34f732877f --- /dev/null +++ b/pkg/agent/routes/record.go @@ -0,0 +1,214 @@ +// Package routes defines the routes for the agent service. +package routes + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" + "go.keploy.io/server/v2/pkg/service/agent" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type AgentRequest struct { + logger *zap.Logger + agent agent.Service +} + +func New(r chi.Router, agent agent.Service, logger *zap.Logger) { + a := &AgentRequest{ + logger: logger, + agent: agent, + } + r.Route("/agent", func(r chi.Router) { + r.Get("/health", a.Health) + r.Post("/incoming", a.HandleIncoming) + r.Post("/outgoing", a.HandleOutgoing) + r.Post("/mock", a.MockOutgoing) + r.Post("/setmocks", a.SetMocks) + r.Post("/register", a.RegisterClients) + r.Get("/consumedmocks", a.GetConsumedMocks) + r.Post("/unregister", a.DeRegisterClients) + }) + +} + +func (a *AgentRequest) Health(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + render.JSON(w, r, "OK") +} + +func (a *AgentRequest) HandleIncoming(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("Cache-Control", "no-cache") + + // Flush headers to ensure the client gets the response immediately + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Create a context with the request's context to manage cancellation + errGrp, _ := errgroup.WithContext(r.Context()) + ctx := context.WithValue(r.Context(), models.ErrGroupKey, errGrp) + + // decode request body + var incomingReq models.IncomingReq + err := json.NewDecoder(r.Body).Decode(&incomingReq) + if err != nil { + http.Error(w, "Error decoding request", http.StatusBadRequest) + return + } + + // Call GetIncoming to get the channel + tc, err := a.agent.GetIncoming(ctx, incomingReq.ClientID, incomingReq.IncomingOptions) + if err != nil { + http.Error(w, "Error retrieving test cases", http.StatusInternalServerError) + return + } + + // Keep the connection alive and stream data + for t := range tc { + select { + case <-r.Context().Done(): + // Client closed the connection or context was cancelled + return + default: + // Stream each test case as JSON + fmt.Printf("Sending Test case: %v\n", t) + render.JSON(w, r, t) + flusher.Flush() // Immediately send data to the client + } + } +} + +func (a *AgentRequest) HandleOutgoing(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("Cache-Control", "no-cache") + + // Flush headers to ensure the client gets the response immediately + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Create a context with the request's context to manage cancellation + errGrp, _ := errgroup.WithContext(r.Context()) + ctx := context.WithValue(r.Context(), models.ErrGroupKey, errGrp) + + var outgoingReq models.OutgoingReq + err := json.NewDecoder(r.Body).Decode(&outgoingReq) + if err != nil { + http.Error(w, "Error decoding request", http.StatusBadRequest) + return + } + + // Call GetOutgoing to get the channel + mockChan, err := a.agent.GetOutgoing(ctx, outgoingReq.ClientID, outgoingReq.OutgoingOptions) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + for m := range mockChan { + select { + case <-r.Context().Done(): + fmt.Println("Context done in HandleOutgoing") + if m != nil { + render.JSON(w, r, m) + flusher.Flush() + } else { + render.JSON(w, r, "No more mocks") + flusher.Flush() + } + return + default: + // Stream each mock as JSON + render.JSON(w, r, m) + flusher.Flush() + } + } +} + +func (a *AgentRequest) RegisterClients(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var registerReq models.RegisterReq + err := json.NewDecoder(r.Body).Decode(®isterReq) + + register := models.AgentResp{ + ClientID: registerReq.SetupOptions.ClientID, + Error: nil, + } + + if err != nil { + register.Error = err + render.JSON(w, r, register) + render.Status(r, http.StatusBadRequest) + return + } + + fmt.Printf("SetupRequest: %v\n", registerReq.SetupOptions.ClientNsPid) + + if registerReq.SetupOptions.ClientNsPid == 0 { + register.Error = fmt.Errorf("Client pid is required") + render.JSON(w, r, register) + render.Status(r, http.StatusBadRequest) + return + } + fmt.Printf("Register Client req: %v\n", registerReq.SetupOptions) + + err = a.agent.RegisterClient(r.Context(), registerReq.SetupOptions) + if err != nil { + register.Error = err + render.JSON(w, r, register) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, register) + render.Status(r, http.StatusOK) +} + +func (a *AgentRequest) DeRegisterClients(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var UnregisterReq models.UnregisterReq + err := json.NewDecoder(r.Body).Decode(&UnregisterReq) + + mockRes := models.AgentResp{ + ClientID: UnregisterReq.ClientID, + Error: nil, + IsSuccess: true, + } + + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, mockRes) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.DeRegisterClient(r.Context(), UnregisterReq) + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, "Client De-registered") +} diff --git a/pkg/agent/routes/replay.go b/pkg/agent/routes/replay.go new file mode 100644 index 0000000000..a06776577b --- /dev/null +++ b/pkg/agent/routes/replay.go @@ -0,0 +1,101 @@ +// Package routes defines the routes for the agent to mock outgoing requests, set mocks and get consumed mocks. +package routes + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" +) + +func (a *AgentRequest) MockOutgoing(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var OutgoingReq models.OutgoingReq + err := json.NewDecoder(r.Body).Decode(&OutgoingReq) + + mockRes := models.AgentResp{ + ClientID: OutgoingReq.ClientID, + Error: nil, + IsSuccess: true, + } + + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, mockRes) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.MockOutgoing(r.Context(), OutgoingReq.ClientID, OutgoingReq.OutgoingOptions) + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, mockRes) + render.Status(r, http.StatusOK) +} + +func (a *AgentRequest) SetMocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var SetMocksReq models.SetMocksReq + err := json.NewDecoder(r.Body).Decode(&SetMocksReq) + + setmockRes := models.AgentResp{ + ClientID: SetMocksReq.ClientID, + Error: nil, + } + if err != nil { + setmockRes.Error = err + setmockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.SetMocks(r.Context(), SetMocksReq.ClientID, SetMocksReq.Filtered, SetMocksReq.UnFiltered) + if err != nil { + setmockRes.Error = err + setmockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, setmockRes) + render.Status(r, http.StatusOK) + +} + +func (a *AgentRequest) GetConsumedMocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + clientID := r.URL.Query().Get("id") + + // convert string to uint64 + clientIDInt, err := strconv.ParseUint(clientID, 10, 64) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusBadRequest) + return + } + + consumedMocks, err := a.agent.GetConsumedMocks(r.Context(), clientIDInt) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, consumedMocks) + render.Status(r, http.StatusOK) + +} diff --git a/pkg/core/service.go b/pkg/agent/service.go similarity index 84% rename from pkg/core/service.go rename to pkg/agent/service.go index 27c43d835f..0948c354db 100644 --- a/pkg/core/service.go +++ b/pkg/agent/service.go @@ -1,29 +1,33 @@ //go:build linux -package core +package agent import ( "context" + "fmt" "sync" "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/core/app" - "go.keploy.io/server/v2/pkg/core/hooks/structs" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" + "go.keploy.io/server/v2/pkg/client/app" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/pkg/models" ) type Hooks interface { - AppInfo DestInfo OutgoingInfo Load(ctx context.Context, id uint64, cfg HookCfg) error Record(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) + // send KeployClient Pid + SendKeployClientInfo(clientID uint64, clientInfo structs.ClientInfo) error + DeleteKeployClientInfo(clientID uint64) error + SendClientProxyInfo(clientID uint64, proxyInfo structs.ProxyInfo) error } type HookCfg struct { - AppID uint64 + ClientID uint64 Pid uint32 IsDocker bool KeployIPV4 string @@ -33,7 +37,7 @@ type HookCfg struct { type App interface { Setup(ctx context.Context, opts app.Options) error - Run(ctx context.Context, inodeChan chan uint64, opts app.Options) error + Run(ctx context.Context, opts app.Options) error Kind(ctx context.Context) utils.CmdType KeployIPv4Addr() string } @@ -45,9 +49,11 @@ type Proxy interface { Mock(ctx context.Context, id uint64, opts models.OutgoingOptions) error SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) + MakeClientDeRegisterd(ctx context.Context) error } type ProxyOptions struct { + ProxyPort uint32 // DNSIPv4Addr is the proxy IP returned by the DNS server. default is loopback address DNSIPv4Addr string // DNSIPv6Addr is the proxy IP returned by the DNS server. default is loopback address @@ -59,10 +65,6 @@ type DestInfo interface { Delete(ctx context.Context, srcPort uint16) error } -type AppInfo interface { - SendDockerAppInfo(id uint64, dockerAppInfo structs.DockerAppInfo) error -} - // For keploy test bench type Tester interface { @@ -79,7 +81,7 @@ type OutgoingInfo interface { } type NetworkAddress struct { - AppID uint64 + ClientID uint64 Version uint32 IPv4Addr uint32 IPv6Addr [4]uint32 @@ -97,6 +99,7 @@ func NewSessions() *Sessions { } func (s *Sessions) Get(id uint64) (*Session, bool) { + fmt.Println("Inside Get of Sessions !!", id) v, ok := s.sessions.Load(id) if !ok { return nil, false diff --git a/pkg/core/tester/tester.go b/pkg/agent/tester/tester.go similarity index 95% rename from pkg/core/tester/tester.go rename to pkg/agent/tester/tester.go index 68fbeb0169..54f8055b07 100644 --- a/pkg/core/tester/tester.go +++ b/pkg/agent/tester/tester.go @@ -9,7 +9,7 @@ import ( "fmt" "time" - "go.keploy.io/server/v2/pkg/core" + "go.keploy.io/server/v2/pkg/agent" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -17,10 +17,10 @@ import ( type Tester struct { logger *zap.Logger - testBenchInfo core.TestBenchInfo + testBenchInfo agent.TestBenchInfo } -func New(logger *zap.Logger, testBenchInfo core.TestBenchInfo) *Tester { +func New(logger *zap.Logger, testBenchInfo agent.TestBenchInfo) *Tester { return &Tester{ logger: logger, testBenchInfo: testBenchInfo, @@ -89,6 +89,7 @@ func (t *Tester) setupReplay(ctx context.Context) error { func (t *Tester) setupRecord(ctx context.Context) error { + fmt.Println("Inside setupRecord") go func() { defer utils.Recover(t.logger) diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go new file mode 100644 index 0000000000..b395d7442a --- /dev/null +++ b/pkg/agent/utils.go @@ -0,0 +1,2 @@ +// Package agent contains backend for the keploy agent. +package agent diff --git a/pkg/core/app/app.go b/pkg/client/app/app.go similarity index 88% rename from pkg/core/app/app.go rename to pkg/client/app/app.go index 5f40049a04..1ccb0596f5 100644 --- a/pkg/core/app/app.go +++ b/pkg/client/app/app.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows // Package app provides functionality for managing applications. package app @@ -8,6 +8,7 @@ import ( "errors" "fmt" "os/exec" + "strings" "syscall" "time" @@ -15,7 +16,6 @@ import ( "go.keploy.io/server/v2/pkg/models" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" "go.keploy.io/server/v2/pkg/platform/docker" @@ -49,10 +49,9 @@ type App struct { container string containerNetwork string containerIPv4 chan string - keployNetwork string + KeployNetwork string keployContainer string keployIPv4 string - inodeChan chan uint64 EnableTesting bool Mode models.Mode } @@ -65,6 +64,7 @@ type Options struct { DockerNetwork string } +// Setup sets up the application for running. func (a *App) Setup(_ context.Context) error { if utils.IsDockerCmd(a.kind) && isDetachMode(a.logger, a.cmd, a.kind) { @@ -78,12 +78,12 @@ func (a *App) Setup(_ context.Context) error { return err } case utils.DockerCompose: + a.cmd = a.cmd + " --force-recreate" err := a.SetupCompose() if err != nil { return err } default: - // setup native binary } return nil } @@ -117,6 +117,14 @@ func (a *App) SetupDocker() error { utils.LogError(a.logger, err, fmt.Sprintf("failed to inject network:%v to the keploy container", a.containerNetwork)) return err } + + // attaching the init container's PID namespace to the app container + err = a.attachInitPid(context.Background()) + if err != nil { + utils.LogError(a.logger, err, "failed to attach init pid") + return err + } + return nil } @@ -165,7 +173,7 @@ func (a *App) SetupCompose() error { if info == nil { info, err = a.docker.SetKeployNetwork(compose) if err != nil { - utils.LogError(a.logger, nil, "failed to set default network in the compose file", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to set default network in the compose file", zap.String("network", a.KeployNetwork)) return err } composeChanged = true @@ -180,24 +188,33 @@ func (a *App) SetupCompose() error { composeChanged = true } - a.keployNetwork = info.Name - - ok, err = a.docker.NetworkExists(a.keployNetwork) + a.KeployNetwork = info.Name + ok, err = a.docker.NetworkExists(a.KeployNetwork) if err != nil { - utils.LogError(a.logger, nil, "failed to find default network", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to find default network", zap.String("network", a.KeployNetwork)) return err } //if keploy-network doesn't exist locally then create it if !ok { - err = a.docker.CreateNetwork(a.keployNetwork) + err = a.docker.CreateNetwork(a.KeployNetwork) if err != nil { - utils.LogError(a.logger, nil, "failed to create default network", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to create default network", zap.String("network", a.KeployNetwork)) return err } } + //check if compose file has keploy-init container + // adding keploy init pid to the compose file + err = a.docker.SetInitPid(compose, a.container) + if err != nil { + utils.LogError(a.logger, nil, "failed to set init pid in the compose file") + return err + } + composeChanged = true + if composeChanged { + err = a.docker.WriteComposeFile(compose, newPath) if err != nil { utils.LogError(a.logger, nil, "failed to write the compose file", zap.String("path", newPath)) @@ -208,7 +225,7 @@ func (a *App) SetupCompose() error { } if a.containerNetwork == "" { - a.containerNetwork = a.keployNetwork + a.containerNetwork = a.KeployNetwork } err = a.injectNetwork(a.containerNetwork) if err != nil { @@ -233,7 +250,7 @@ func (a *App) injectNetwork(network string) error { return err } - a.keployNetwork = network + a.KeployNetwork = network //sending new proxy ip to kernel, since dynamically injected new network has different ip for keploy. inspect, err := a.docker.ContainerInspect(context.Background(), a.keployContainer) @@ -248,7 +265,7 @@ func (a *App) injectNetwork(network string) error { //TODO: check the logic for correctness for n, settings := range keployNetworks { if n == network { - a.keployIPv4 = settings.IPAddress + a.keployIPv4 = settings.IPAddress // TODO: keployIPv4 needs to be send to the agent a.logger.Info("Successfully injected network to the keploy container", zap.Any("Keploy container", a.keployContainer), zap.Any("appNetwork", network), zap.String("keploy container ip", a.keployIPv4)) return nil } @@ -261,6 +278,26 @@ func (a *App) injectNetwork(network string) error { return fmt.Errorf("failed to find the network:%v in the keploy container", network) } +// AttachInitPid modifies the existing Docker command to attach the init container's PID namespace +func (a *App) attachInitPid(_ context.Context) error { + if a.cmd == "" { + return fmt.Errorf("no command provided to modify") + } + + // Add the --pid=container: flag to the command + pidMode := fmt.Sprintf("--pid=container:%s", "keploy-init") + // Inject the pidMode flag after 'docker run' in the command + parts := strings.SplitN(a.cmd, " ", 3) // Split by first two spaces to isolate "docker run" + if len(parts) < 3 { + return fmt.Errorf("invalid command structure: %s", a.cmd) + } + + // Modify the command to insert the pidMode + a.cmd = fmt.Sprintf("%s %s %s %s", parts[0], parts[1], pidMode, parts[2]) + + return nil +} + func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { if e.Action != "start" { return false, nil @@ -290,8 +327,7 @@ func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { return false, err } - a.inodeChan <- inode - a.logger.Debug("container started and successfully extracted inode", zap.Any("inode", inode)) + a.logger.Info("container started and successfully extracted inode", zap.Any("inode", inode)) if info.NetworkSettings == nil || info.NetworkSettings.Networks == nil { a.logger.Debug("container network settings not available", zap.Any("containerDetails.NetworkSettings", info.NetworkSettings)) return false, nil @@ -309,7 +345,6 @@ func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { func (a *App) getDockerMeta(ctx context.Context) <-chan error { // listen for the docker daemon events defer a.logger.Debug("exiting from goroutine of docker daemon event listener") - errCh := make(chan error, 1) timer := time.NewTimer(time.Duration(a.containerDelay) * time.Second) logTicker := time.NewTicker(1 * time.Second) @@ -323,7 +358,7 @@ func (a *App) getDockerMeta(ctx context.Context) <-chan error { filters.KeyValuePair{Key: "action", Value: "start"}, ) - messages, errCh2 := a.docker.Events(ctx, types.EventsOptions{ + messages, errCh2 := a.docker.Events(ctx, events.ListOptions{ Filters: eventFilter, }) @@ -411,12 +446,12 @@ func (a *App) runDocker(ctx context.Context) models.AppError { } return models.AppError{AppErrorType: models.ErrInternal, Err: err} case <-ctx.Done(): + fmt.Println("ctx.Done called in runDocker") return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: ctx.Err()} } } -func (a *App) Run(ctx context.Context, inodeChan chan uint64) models.AppError { - a.inodeChan = inodeChan +func (a *App) Run(ctx context.Context) models.AppError { if utils.IsDockerCmd(a.kind) { return a.runDocker(ctx) @@ -464,7 +499,7 @@ func (a *App) run(ctx context.Context) models.AppError { cmdCancel := func(cmd *exec.Cmd) func() error { return func() error { if utils.IsDockerCmd(a.kind) { - a.logger.Debug("sending SIGINT to the container", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) + a.logger.Info("sending SIGINT to the container", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) err := utils.SendSignal(a.logger, -cmd.Process.Pid, syscall.SIGINT) return err } @@ -473,6 +508,8 @@ func (a *App) run(ctx context.Context) models.AppError { } var err error + fmt.Println("userCmd", userCmd) + cmdErr := utils.ExecuteCommand(ctx, a.logger, userCmd, cmdCancel, 25*time.Second) if cmdErr.Err != nil { switch cmdErr.Type { diff --git a/pkg/core/app/util.go b/pkg/client/app/util.go similarity index 99% rename from pkg/core/app/util.go rename to pkg/client/app/util.go index 6d374fe159..b72bac6964 100644 --- a/pkg/core/app/util.go +++ b/pkg/client/app/util.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package app diff --git a/pkg/core/core_linux.go b/pkg/core/core_linux.go deleted file mode 100644 index 1db78c78f7..0000000000 --- a/pkg/core/core_linux.go +++ /dev/null @@ -1,272 +0,0 @@ -//go:build linux - -// Package core provides functionality for managing core functionalities in Keploy. -package core - -import ( - "context" - "errors" - "fmt" - "sync" - - "golang.org/x/sync/errgroup" - - "go.keploy.io/server/v2/pkg/core/app" - "go.keploy.io/server/v2/pkg/core/hooks/structs" - "go.keploy.io/server/v2/pkg/models" - "go.keploy.io/server/v2/pkg/platform/docker" - "go.keploy.io/server/v2/utils" - "go.uber.org/zap" -) - -type Core struct { - Proxy // embedding the Proxy interface to transfer the proxy methods to the core object - Hooks // embedding the Hooks interface to transfer the hooks methods to the core object - Tester // embedding the Tester interface to transfer the tester methods to the core object - dockerClient docker.Client //embedding the docker client to transfer the docker client methods to the core object - logger *zap.Logger - id utils.AutoInc - apps sync.Map - proxyStarted bool -} - -func New(logger *zap.Logger, hook Hooks, proxy Proxy, tester Tester, client docker.Client) *Core { - return &Core{ - logger: logger, - Hooks: hook, - Proxy: proxy, - Tester: tester, - dockerClient: client, - } -} - -func (c *Core) Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) { - // create a new app and store it in the map - id := uint64(c.id.Next()) - a := app.NewApp(c.logger, id, cmd, c.dockerClient, app.Options{ - DockerNetwork: opts.DockerNetwork, - Container: opts.Container, - DockerDelay: opts.DockerDelay, - }) - c.apps.Store(id, a) - - err := a.Setup(ctx) - if err != nil { - utils.LogError(c.logger, err, "failed to setup app") - return 0, err - } - return id, nil -} - -func (c *Core) getApp(id uint64) (*app.App, error) { - a, ok := c.apps.Load(id) - if !ok { - return nil, fmt.Errorf("app with id:%v not found", id) - } - - // type assertion on the app - h, ok := a.(*app.App) - if !ok { - return nil, fmt.Errorf("failed to type assert app with id:%v", id) - } - - return h, nil -} - -func (c *Core) Hook(ctx context.Context, id uint64, opts models.HookOptions) error { - hookErr := errors.New("failed to hook into the app") - - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return hookErr - } - - isDocker := false - appKind := a.Kind(ctx) - //check if the app is docker/docker-compose or native - if utils.IsDockerCmd(appKind) { - isDocker = true - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) - if !ok { - return errors.New("failed to get the error group from the context") - } - - // create a new error group for the hooks - hookErrGrp, _ := errgroup.WithContext(ctx) - hookCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the hookCtx to control the lifecycle of the hooks - hookCtx, hookCtxCancel := context.WithCancel(hookCtx) - hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) - - // create a new error group for the proxy - proxyErrGrp, _ := errgroup.WithContext(ctx) - proxyCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the proxyCtx to control the lifecycle of the proxy - proxyCtx, proxyCtxCancel := context.WithCancel(proxyCtx) - proxyCtx = context.WithValue(proxyCtx, models.ErrGroupKey, proxyErrGrp) - - g.Go(func() error { - <-ctx.Done() - - proxyCtxCancel() - err = proxyErrGrp.Wait() - if err != nil { - utils.LogError(c.logger, err, "failed to stop the proxy") - } - - hookCtxCancel() - err := hookErrGrp.Wait() - if err != nil { - utils.LogError(c.logger, err, "failed to unload the hooks") - } - - //deleting in order to free the memory in case of rerecord. otherwise different app id will be created for the same app. - c.apps.Delete(id) - c.id = utils.AutoInc{} - - return nil - }) - - //load hooks - err = c.Hooks.Load(hookCtx, id, HookCfg{ - AppID: id, - Pid: 0, - IsDocker: isDocker, - KeployIPV4: a.KeployIPv4Addr(), - Mode: opts.Mode, - Rules: opts.Rules, - }) - if err != nil { - utils.LogError(c.logger, err, "failed to load hooks") - return hookErr - } - - if c.proxyStarted { - c.logger.Debug("Proxy already started") - // return nil - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - // TODO: Hooks can be loaded multiple times but proxy should be started only once - // if there is another containerized app, then we need to pass new (ip:port) of proxy to the eBPF - // as the network namespace is different for each container and so is the keploy/proxy IP to communicate with the app. - // start proxy - err = c.Proxy.StartProxy(proxyCtx, ProxyOptions{ - DNSIPv4Addr: a.KeployIPv4Addr(), - //DnsIPv6Addr: "" - }) - if err != nil { - utils.LogError(c.logger, err, "failed to start proxy") - return hookErr - } - - c.proxyStarted = true - - // For keploy test bench - if opts.EnableTesting { - - // enable testing in the app - a.EnableTesting = true - a.Mode = opts.Mode - - // Setting up the test bench - err := c.Tester.Setup(ctx, models.TestingOptions{Mode: opts.Mode}) - if err != nil { - utils.LogError(c.logger, err, "error while setting up the test bench environment") - return errors.New("failed to setup the test bench") - } - } - - return nil -} - -func (c *Core) Run(ctx context.Context, id uint64, _ models.RunOptions) models.AppError { - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return models.AppError{AppErrorType: models.ErrInternal, Err: err} - } - - runAppErrGrp, runAppCtx := errgroup.WithContext(ctx) - - inodeErrCh := make(chan error, 1) - appErrCh := make(chan models.AppError, 1) - inodeChan := make(chan uint64, 1) //send inode to the hook - - defer func() { - err := runAppErrGrp.Wait() - defer close(inodeErrCh) - defer close(inodeChan) - if err != nil { - utils.LogError(c.logger, err, "failed to stop the app") - } - }() - - runAppErrGrp.Go(func() error { - defer utils.Recover(c.logger) - if a.Kind(ctx) == utils.Native { - return nil - } - select { - case inode := <-inodeChan: - err := c.Hooks.SendDockerAppInfo(id, structs.DockerAppInfo{AppInode: inode, ClientID: id}) - if err != nil { - utils.LogError(c.logger, err, "") - - inodeErrCh <- errors.New("failed to send inode to the kernel") - } - case <-ctx.Done(): - return nil - } - return nil - }) - - runAppErrGrp.Go(func() error { - defer utils.Recover(c.logger) - defer close(appErrCh) - appErr := a.Run(runAppCtx, inodeChan) - if appErr.Err != nil { - utils.LogError(c.logger, appErr.Err, "error while running the app") - appErrCh <- appErr - } - return nil - }) - - select { - case <-runAppCtx.Done(): - return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: nil} - case appErr := <-appErrCh: - return appErr - case inodeErr := <-inodeErrCh: - return models.AppError{AppErrorType: models.ErrInternal, Err: inodeErr} - } -} - -func (c *Core) GetContainerIP(_ context.Context, id uint64) (string, error) { - - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return "", err - } - - ip := a.ContainerIPv4Addr() - c.logger.Debug("ip address of the target app container", zap.Any("ip", ip)) - if ip == "" { - return "", fmt.Errorf("failed to get the IP address of the app container. Try increasing --delay (in seconds)") - } - - return ip, nil -} diff --git a/pkg/core/hooks/bpf_arm64_bpfel.o b/pkg/core/hooks/bpf_arm64_bpfel.o deleted file mode 100644 index f8c9fe223f..0000000000 Binary files a/pkg/core/hooks/bpf_arm64_bpfel.o and /dev/null differ diff --git a/pkg/core/hooks/bpf_x86_bpfel.o b/pkg/core/hooks/bpf_x86_bpfel.o deleted file mode 100644 index f3f76af725..0000000000 Binary files a/pkg/core/hooks/bpf_x86_bpfel.o and /dev/null differ diff --git a/pkg/core/proxy/parsers.go b/pkg/core/proxy/parsers.go deleted file mode 100644 index aacc44e513..0000000000 --- a/pkg/core/proxy/parsers.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build linux - -package proxy - -import ( - // import all the integrations - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/generic" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/grpc" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/http" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/mongo" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/postgres/v1" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/redis" -) diff --git a/pkg/core/record.go b/pkg/core/record.go deleted file mode 100644 index 471ebf40e5..0000000000 --- a/pkg/core/record.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build linux - -package core - -import ( - "context" - - "go.keploy.io/server/v2/pkg/models" -) - -func (c *Core) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - return c.Hooks.Record(ctx, id, opts) -} - -func (c *Core) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { - m := make(chan *models.Mock, 500) - - err := c.Proxy.Record(ctx, id, m, opts) - if err != nil { - return nil, err - } - - return m, nil -} diff --git a/pkg/core/replay.go b/pkg/core/replay.go deleted file mode 100644 index 4bc6cfdd6e..0000000000 --- a/pkg/core/replay.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build linux - -package core - -import ( - "context" - - "go.keploy.io/server/v2/pkg/models" -) - -func (c *Core) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { - - err := c.Proxy.Mock(ctx, id, opts) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/models/agent.go b/pkg/models/agent.go new file mode 100644 index 0000000000..6c3df53584 --- /dev/null +++ b/pkg/models/agent.go @@ -0,0 +1,36 @@ +package models + +type OutgoingReq struct { + OutgoingOptions OutgoingOptions `json:"outgoingOptions"` + ClientID uint64 `json:"clientId"` +} + +type IncomingReq struct { + IncomingOptions IncomingOptions `json:"incomingOptions"` + ClientID uint64 `json:"clientId"` +} + +type RegisterReq struct { + SetupOptions SetupOptions `json:"setupOptions"` +} + +type AgentResp struct { + ClientID uint64 `json:"clientID"` // uuid of the app + Error error `json:"error"` + IsSuccess bool `json:"isSuccess"` +} + +type RunReq struct { + RunOptions RunOptions `json:"runOptions"` + ClientID uint64 `json:"clientId"` +} + +type SetMocksReq struct { + Filtered []*Mock `json:"filtered"` + UnFiltered []*Mock `json:"unFiltered"` + ClientID uint64 `json:"clientId"` +} +type UnregisterReq struct { + ClientID uint64 `json:"clientId"` + Mode Mode `json:"mode"` +} diff --git a/pkg/models/instrument.go b/pkg/models/instrument.go index 15a6dad31b..f4b7e6fa28 100644 --- a/pkg/models/instrument.go +++ b/pkg/models/instrument.go @@ -11,16 +11,19 @@ type HookOptions struct { Rules []config.BypassRule Mode Mode EnableTesting bool + IsDocker bool + ProxyPort uint32 + ServerPort uint32 } type OutgoingOptions struct { Rules []config.BypassRule MongoPassword string // TODO: role of SQLDelay should be mentioned in the comments. - SQLDelay time.Duration // This is the same as Application delay. - FallBackOnMiss bool // this enables to pass the request to the actual server if no mock is found during test mode. - Mocking bool // used to enable/disable mocking - DstCfg *ConditionalDstCfg + SQLDelay time.Duration // This is the same as Application delay. + FallBackOnMiss bool // this enables to pass the request to the actual server if no mock is found during test mode. + Mocking bool // used to enable/disable mocking + DstCfg *ConditionalDstCfg `json:"-"` } type ConditionalDstCfg struct { @@ -34,9 +37,19 @@ type IncomingOptions struct { } type SetupOptions struct { + ClientID uint64 Container string DockerNetwork string DockerDelay uint64 + ClientNsPid uint32 + ClientInode uint64 + AppInode uint64 + Cmd string + IsDocker bool + EnableTesting bool + ProxyPort uint32 + CommandType string + Mode Mode } type RunOptions struct { diff --git a/pkg/models/mock.go b/pkg/models/mock.go index d0d0838df6..42e0456abd 100755 --- a/pkg/models/mock.go +++ b/pkg/models/mock.go @@ -6,6 +6,20 @@ import ( "go.keploy.io/server/v2/pkg/models/mysql" ) +type MockResponse struct { + Type ResponseType `json:"Type,omitempty" bson:"Type,omitempty"` + Error error + Mock *Mock +} + +type ResponseType string + +const ( + EOF ResponseType = "EOF" + PACKET ResponseType = "PACKET" + ERROR ResponseType = "ERROR" +) + type Mock struct { Version Version `json:"Version,omitempty" bson:"Version,omitempty"` Name string `json:"Name,omitempty" bson:"Name,omitempty"` diff --git a/pkg/models/mysql/comm.go b/pkg/models/mysql/comm.go index c0ed297ca8..2dfef943f4 100644 --- a/pkg/models/mysql/comm.go +++ b/pkg/models/mysql/comm.go @@ -1,4 +1,4 @@ -// Package mysql in models provides realted structs for mysql protocol +// Package mysql in models provides related structs for mysql protocol package mysql // This file contains struct for command phase packets @@ -13,157 +13,157 @@ package mysql // COM_QUERY packet (currently does not support if CLIENT_QUERY_ATTRIBUTES is set) type QueryPacket struct { - Command byte `yaml:"command"` - Query string `yaml:"query"` + Command byte `yaml:"command" json:"command"` + Query string `yaml:"query" json:"query"` } // LocalInFileRequestPacket is used to send local file request to server, currently not supported type LocalInFileRequestPacket struct { - PacketType byte `yaml:"command"` - Filename string + PacketType byte `yaml:"command" json:"command"` + Filename string `yaml:"filename" json:"filename"` } // TextResultSet is used as a response packet for COM_QUERY type TextResultSet struct { - ColumnCount uint64 `yaml:"columnCount"` - Columns []*ColumnDefinition41 `yaml:"columns"` - EOFAfterColumns []byte `yaml:"eofAfterColumns"` - Rows []*TextRow `yaml:"rows"` - FinalResponse *GenericResponse `yaml:"FinalResponse"` + ColumnCount uint64 `yaml:"columnCount" json:"columnCount"` + Columns []*ColumnDefinition41 `yaml:"columns" json:"columns"` + EOFAfterColumns []byte `yaml:"eofAfterColumns" json:"eofAfterColumns"` + Rows []*TextRow `yaml:"rows" json:"rows"` + FinalResponse *GenericResponse `yaml:"FinalResponse" json:"FinalResponse"` } // BinaryProtocolResultSet is used as a response packet for COM_STMT_EXECUTE type BinaryProtocolResultSet struct { - ColumnCount uint64 `yaml:"columnCount"` - Columns []*ColumnDefinition41 `yaml:"columns"` - EOFAfterColumns []byte `yaml:"eofAfterColumns"` - Rows []*BinaryRow `yaml:"rows"` - FinalResponse *GenericResponse `yaml:"FinalResponse"` + ColumnCount uint64 `yaml:"columnCount" json:"columnCount"` + Columns []*ColumnDefinition41 `yaml:"columns" json:"columns"` + EOFAfterColumns []byte `yaml:"eofAfterColumns" json:"eofAfterColumns"` + Rows []*BinaryRow `yaml:"rows" json:"rows"` + FinalResponse *GenericResponse `yaml:"FinalResponse" json:"FinalResponse"` } type GenericResponse struct { - Data []byte `yaml:"data"` - Type string `yaml:"type"` + Data []byte `yaml:"data" json:"data"` + Type string `yaml:"type" json:"type"` } // Columns type ColumnCount struct { - // Header Header `yaml:"header"` - Count uint64 `yaml:"count"` + // Header Header `yaml:"header" json:"header"` + Count uint64 `yaml:"count" json:"count"` } type ColumnDefinition41 struct { - Header Header `yaml:"header"` - Catalog string `yaml:"catalog"` - Schema string `yaml:"schema"` - Table string `yaml:"table"` - OrgTable string `yaml:"org_table"` - Name string `yaml:"name"` - OrgName string `yaml:"org_name"` - FixedLength byte `yaml:"fixed_length"` - CharacterSet uint16 `yaml:"character_set"` - ColumnLength uint32 `yaml:"column_length"` - Type byte `yaml:"type"` - Flags uint16 `yaml:"flags"` - Decimals byte `yaml:"decimals"` - Filler []byte `yaml:"filler"` - DefaultValue string `yaml:"defaultValue"` -} - -//Rows + Header Header `yaml:"header" json:"header"` + Catalog string `yaml:"catalog" json:"catalog"` + Schema string `yaml:"schema" json:"schema"` + Table string `yaml:"table" json:"table"` + OrgTable string `yaml:"org_table" json:"org_table"` + Name string `yaml:"name" json:"name"` + OrgName string `yaml:"org_name" json:"org_name"` + FixedLength byte `yaml:"fixed_length" json:"fixed_length"` + CharacterSet uint16 `yaml:"character_set" json:"character_set"` + ColumnLength uint32 `yaml:"column_length" json:"column_length"` + Type byte `yaml:"type" json:"type"` + Flags uint16 `yaml:"flags" json:"flags"` + Decimals byte `yaml:"decimals" json:"decimals"` + Filler []byte `yaml:"filler" json:"filler"` + DefaultValue string `yaml:"defaultValue" json:"defaultValue"` +} + +// Rows type TextRow struct { - Header Header `yaml:"header"` - Values []ColumnEntry `yaml:"values"` + Header Header `yaml:"header" json:"header"` + Values []ColumnEntry `yaml:"values" json:"values"` } type BinaryRow struct { - Header Header `yaml:"header"` - Values []ColumnEntry `yaml:"values"` - OkAfterRow bool `yaml:"okAfterRow"` - RowNullBuffer []byte `yaml:"rowNullBuffer"` + Header Header `yaml:"header" json:"header"` + Values []ColumnEntry `yaml:"values" json:"values"` + OkAfterRow bool `yaml:"okAfterRow" json:"okAfterRow"` + RowNullBuffer []byte `yaml:"rowNullBuffer" json:"rowNullBuffer"` } type ColumnEntry struct { - Type FieldType `yaml:"type"` - Name string `yaml:"name"` - Value interface{} `yaml:"value"` - Unsigned bool `yaml:"unsigned"` + Type FieldType `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` + Value interface{} `yaml:"value" json:"value"` + Unsigned bool `yaml:"unsigned" json:"unsigned"` } // COM_STMT_PREPARE packet type StmtPreparePacket struct { - Command byte `yaml:"command"` - Query string `yaml:"query"` + Command byte `yaml:"command" json:"command"` + Query string `yaml:"query" json:"query"` } // COM_STMT_PREPARE_OK packet type StmtPrepareOkPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - NumColumns uint16 `yaml:"num_columns"` - NumParams uint16 `yaml:"num_params"` - Filler byte `yaml:"filler"` - WarningCount uint16 `yaml:"warning_count"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + NumColumns uint16 `yaml:"num_columns" json:"num_columns"` + NumParams uint16 `yaml:"num_params" json:"num_params"` + Filler byte `yaml:"filler" json:"filler"` + WarningCount uint16 `yaml:"warning_count" json:"warning_count"` - ParamDefs []*ColumnDefinition41 `yaml:"param_definitions"` - EOFAfterParamDefs []byte `yaml:"eofAfterParamDefs"` - ColumnDefs []*ColumnDefinition41 `yaml:"column_definitions"` - EOFAfterColumnDefs []byte `yaml:"eofAfterColumnDefs"` + ParamDefs []*ColumnDefinition41 `yaml:"param_definitions" json:"param_definitions"` + EOFAfterParamDefs []byte `yaml:"eofAfterParamDefs" json:"eofAfterParamDefs"` + ColumnDefs []*ColumnDefinition41 `yaml:"column_definitions" json:"column_definitions"` + EOFAfterColumnDefs []byte `yaml:"eofAfterColumnDefs" json:"eofAfterColumnDefs"` } // COM_STMT_EXECUTE packet type StmtExecutePacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - Flags byte `yaml:"flags"` - IterationCount uint32 `yaml:"iteration_count"` - ParameterCount int `yaml:"parameter_count"` - NullBitmap []byte `yaml:"null_bitmap"` - NewParamsBindFlag byte `yaml:"new_params_bind_flag"` - Parameters []Parameter `yaml:"parameters"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + Flags byte `yaml:"flags" json:"flags"` + IterationCount uint32 `yaml:"iteration_count" json:"iteration_count"` + ParameterCount int `yaml:"parameter_count" json:"parameter_count"` + NullBitmap []byte `yaml:"null_bitmap" json:"null_bitmap"` + NewParamsBindFlag byte `yaml:"new_params_bind_flag" json:"new_params_bind_flag"` + Parameters []Parameter `yaml:"parameters" json:"parameters"` } type Parameter struct { - Type uint16 `yaml:"type"` - Unsigned bool `yaml:"unsigned"` - Name string `yaml:"name,omitempty"` - Value []byte `yaml:"value"` + Type uint16 `yaml:"type" json:"type"` + Unsigned bool `yaml:"unsigned" json:"unsigned"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Value []byte `yaml:"value" json:"value"` } // COM_STMT_FETCH packet is not currently supported because its response involves multi-resultset type StmtFetchPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - NumRows uint32 `yaml:"num_rows"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + NumRows uint32 `yaml:"num_rows" json:"num_rows"` } // COM_STMT_CLOSE packet type StmtClosePacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` } // COM_STMT_RESET packet type StmtResetPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` } // COM_STMT_SEND_LONG_DATA packet type StmtSendLongDataPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - ParameterID uint16 `yaml:"parameter_id"` - Data []byte `yaml:"data"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + ParameterID uint16 `yaml:"parameter_id" json:"parameter_id"` + Data []byte `yaml:"data" json:"data"` } // Utility commands @@ -171,51 +171,51 @@ type StmtSendLongDataPacket struct { // COM_QUIT packet type QuitPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_INIT_DB packet type InitDBPacket struct { - Command byte `yaml:"command"` - Schema string `yaml:"schema"` + Command byte `yaml:"command" json:"command"` + Schema string `yaml:"schema" json:"schema"` } // COM_STATISTICS packet type StatisticsPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_DEBUG packet type DebugPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_PING packet type PingPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_RESET_CONNECTION packet type ResetConnectionPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_SET_OPTION packet type SetOptionPacket struct { - Status byte `yaml:"status"` - Option uint16 `yaml:"option"` + Status byte `yaml:"status" json:"status"` + Option uint16 `yaml:"option" json:"option"` } // COM_CHANGE_USER packet (Not completed/supported as of now) //refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_change_user.html type ChangeUserPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` // rest of the fields are not present as the packet is not supported } diff --git a/pkg/models/mysql/conn.go b/pkg/models/mysql/conn.go index 51af3a66dd..5045a70190 100644 --- a/pkg/models/mysql/conn.go +++ b/pkg/models/mysql/conn.go @@ -7,29 +7,29 @@ package mysql // HandshakeV10Packet represents the initial handshake packet sent by the server to the client type HandshakeV10Packet struct { - ProtocolVersion uint8 `yaml:"protocol_version"` - ServerVersion string `yaml:"server_version"` - ConnectionID uint32 `yaml:"connection_id"` - AuthPluginData []byte `yaml:"auth_plugin_data,omitempty,flow"` - Filler byte `yaml:"filler"` - CapabilityFlags uint32 `yaml:"capability_flags"` - CharacterSet uint8 `yaml:"character_set"` - StatusFlags uint16 `yaml:"status_flags"` - AuthPluginName string `yaml:"auth_plugin_name"` + ProtocolVersion uint8 `yaml:"protocol_version" json:"protocol_version"` + ServerVersion string `yaml:"server_version" json:"server_version"` + ConnectionID uint32 `yaml:"connection_id" json:"connection_id"` + AuthPluginData []byte `yaml:"auth_plugin_data,omitempty,flow" json:"auth_plugin_data,omitempty"` + Filler byte `yaml:"filler" json:"filler"` + CapabilityFlags uint32 `yaml:"capability_flags" json:"capability_flags"` + CharacterSet uint8 `yaml:"character_set" json:"character_set"` + StatusFlags uint16 `yaml:"status_flags" json:"status_flags"` + AuthPluginName string `yaml:"auth_plugin_name" json:"auth_plugin_name"` } // HandshakeResponse41Packet represents the response packet sent by the client to the server after receiving the HandshakeV10Packet type HandshakeResponse41Packet struct { - CapabilityFlags uint32 `yaml:"capability_flags"` - MaxPacketSize uint32 `yaml:"max_packet_size"` - CharacterSet uint8 `yaml:"character_set"` - Filler [23]byte `yaml:"filler,omitempty,flow"` - Username string `yaml:"username"` - AuthResponse []byte `yaml:"auth_response,omitempty,flow"` - Database string `yaml:"database"` - AuthPluginName string `yaml:"auth_plugin_name"` - ConnectionAttributes map[string]string `yaml:"connection_attributes,omitempty"` - ZstdCompressionLevel byte `yaml:"zstdcompressionlevel"` + CapabilityFlags uint32 `yaml:"capability_flags" json:"capability_flags"` + MaxPacketSize uint32 `yaml:"max_packet_size" json:"max_packet_size"` + CharacterSet uint8 `yaml:"character_set" json:"character_set"` + Filler [23]byte `yaml:"filler,omitempty,flow" json:"filler,omitempty"` + Username string `yaml:"username" json:"username"` + AuthResponse []byte `yaml:"auth_response,omitempty,flow" json:"auth_response,omitempty"` + Database string `yaml:"database" json:"database"` + AuthPluginName string `yaml:"auth_plugin_name" json:"auth_plugin_name"` + ConnectionAttributes map[string]string `yaml:"connection_attributes,omitempty" json:"connection_attributes,omitempty"` + ZstdCompressionLevel byte `yaml:"zstdcompressionlevel" json:"zstdcompressionlevel"` } type SSLRequestPacket struct { @@ -43,26 +43,26 @@ type SSLRequestPacket struct { // AuthSwitchRequestPacket represents the packet sent by the server to the client to switch to a different authentication method type AuthSwitchRequestPacket struct { - StatusTag byte `yaml:"status_tag"` - PluginName string `yaml:"plugin_name"` - PluginData string `yaml:"plugin_data"` + StatusTag byte `yaml:"status_tag" json:"status_tag"` + PluginName string `yaml:"plugin_name" json:"plugin_name"` + PluginData string `yaml:"plugin_data" json:"plugin_data"` } // AuthSwitchResponsePacket represents the packet sent by the client to the server in response to an AuthSwitchRequestPacket. // Note: If the server sends an AuthMoreDataPacket, the client will continue sending AuthSwitchResponsePackets until the server sends an OK packet or an ERR packet. type AuthSwitchResponsePacket struct { - Data string `yaml:"data"` + Data string `yaml:"data" json:"data"` } // AuthMoreDataPacket represents the packet sent by the server to the client to request additional data for authentication type AuthMoreDataPacket struct { - StatusTag byte `yaml:"status_tag"` - Data string `yaml:"data"` + StatusTag byte `yaml:"status_tag" json:"status_tag"` + Data string `yaml:"data" json:"data"` } // AuthNextFactorPacket represents the packet sent by the server to the client to request the next factor for multi-factor authentication type AuthNextFactorPacket struct { - PacketType byte `yaml:"packet_type"` - PluginName string `yaml:"plugin_name"` - PluginData string `yaml:"plugin_data"` + PacketType byte `yaml:"packet_type" json:"packet_type"` + PluginName string `yaml:"plugin_name" json:"plugin_name"` + PluginData string `yaml:"plugin_data" json:"plugin_data"` } diff --git a/pkg/models/mysql/generic.go b/pkg/models/mysql/generic.go index 07be522951..358dea3b19 100644 --- a/pkg/models/mysql/generic.go +++ b/pkg/models/mysql/generic.go @@ -1,7 +1,7 @@ package mysql // This file contains structs for mysql generic response packets -//refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_response_packets.html +// refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_response_packets.html // OKPacket represents the OK packet sent by the server to the client, it represents a successful completion of a command type OKPacket struct { @@ -15,16 +15,16 @@ type OKPacket struct { // ERRPacket represents the ERR packet sent by the server to the client, it represents an error occurred during the execution of a command type ERRPacket struct { - Header byte `yaml:"header"` - ErrorCode uint16 `yaml:"error_code"` - SQLStateMarker string `yaml:"sql_state_marker"` - SQLState string `yaml:"sql_state"` - ErrorMessage string `yaml:"error_message"` + Header byte `json:"header" yaml:"header"` + ErrorCode uint16 `json:"error_code" yaml:"error_code"` + SQLStateMarker string `json:"sql_state_marker" yaml:"sql_state_marker"` + SQLState string `json:"sql_state" yaml:"sql_state"` + ErrorMessage string `json:"error_message" yaml:"error_message"` } // EOFPacket represents the EOF packet sent by the server to the client, it represents the end of a query execution result type EOFPacket struct { - Header byte `yaml:"header"` - Warnings uint16 `yaml:"warnings"` - StatusFlags uint16 `yaml:"status_flags"` + Header byte `json:"header" yaml:"header"` + Warnings uint16 `json:"warnings" yaml:"warnings"` + StatusFlags uint16 `json:"status_flags" yaml:"status_flags"` } diff --git a/pkg/models/mysql/mysql.go b/pkg/models/mysql/mysql.go index c1f6389a7e..12c790e1ba 100644 --- a/pkg/models/mysql/mysql.go +++ b/pkg/models/mysql/mysql.go @@ -1,6 +1,8 @@ package mysql import ( + "encoding/json" + "errors" "time" "gopkg.in/yaml.v3" @@ -59,3 +61,299 @@ type Header struct { PayloadLength uint32 `json:"payload_length" yaml:"payload_length"` SequenceID uint8 `json:"sequence_id" yaml:"sequence_id"` } + +// custom marshal and unmarshal methods for Request and Response structs + +// MarshalJSON implements json.Marshaler for Request because of interface type of field 'Message' +func (r *Request) MarshalJSON() ([]byte, error) { + // create an alias struct to avoid infinite recursion + type RequestAlias struct { + Header *PacketInfo `json:"header"` + Message json.RawMessage `json:"message"` + Meta map[string]string `json:"meta,omitempty"` + } + + aux := RequestAlias{ + Header: r.Header, + Message: json.RawMessage(nil), + Meta: r.Meta, + } + + if r.Message != nil { + // Marshal the message interface{} into JSON + msgJSON, err := json.Marshal(r.Message) + if err != nil { + return nil, err + } + aux.Message = msgJSON + } + + // Marshal the alias struct into JSON + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler for Request because of interface type of field 'Message' +func (r *Request) UnmarshalJSON(data []byte) error { + // Alias struct to prevent recursion during unmarshalling + type RequestAlias struct { + Header *PacketInfo `json:"header"` + Message json.RawMessage `json:"message"` + Meta map[string]string `json:"meta,omitempty"` + } + var aux RequestAlias + + // Unmarshal the data into the alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Assign the unmarshalled data to the original struct + r.Header = aux.Header + r.Meta = aux.Meta + + // Unmarshal the message field based on the type in the header + switch r.Header.Type { + case HandshakeResponse41: + var msg HandshakeResponse41Packet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CachingSha2PasswordToString(RequestPublicKey): + var msg string + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = msg + + case "encrypted_password": + var msg string + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = msg + + case CommandStatusToString(COM_QUIT): + var msg QuitPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_INIT_DB): + var msg InitDBPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STATISTICS): + var msg StatisticsPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_DEBUG): + var msg DebugPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_PING): + var msg PingPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_CHANGE_USER): + var msg ChangeUserPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_RESET_CONNECTION): + var msg ResetConnectionPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_QUERY): + var msg QueryPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_PREPARE): + var msg StmtPreparePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_EXECUTE): + var msg StmtExecutePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_CLOSE): + var msg StmtClosePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_RESET): + var msg StmtResetPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_SEND_LONG_DATA): + var msg StmtSendLongDataPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + default: + return errors.New("failed to unmarshal unknown request packet type") + } + return nil +} + +// MarshalJSON implements json.Marshaler for Response because of interface type of field 'Message' +func (r *Response) MarshalJSON() ([]byte, error) { + // Alias to avoid recursion + type ResponseAlias struct { + PacketBundle `json:"packet_bundle"` + Payload string `json:"payload,omitempty"` + Message json.RawMessage `json:"message"` + } + + aux := ResponseAlias{ + PacketBundle: r.PacketBundle, + Payload: r.Payload, + } + + if r.Message != nil { + // Marshal the message interface{} into JSON + msgJSON, err := json.Marshal(r.Message) + if err != nil { + return nil, err + } + aux.Message = msgJSON + } + + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler for Response because of interface type of field 'Message' +func (r *Response) UnmarshalJSON(data []byte) error { + // Alias struct to prevent recursion + type ResponseAlias struct { + PacketBundle `json:"packet_bundle"` + Payload string `json:"payload,omitempty"` + Message json.RawMessage `json:"message"` + } + var aux ResponseAlias + + // Unmarshal the data into the alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Assign the unmarshalled data to the original struct + r.PacketBundle = aux.PacketBundle + r.Payload = aux.Payload + + // Unmarshal the message field based on the type in the header + switch r.PacketBundle.Header.Type { + // Generic response + case StatusToString(EOF): + var msg EOFPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case StatusToString(ERR): + var msg ERRPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case StatusToString(OK): + var msg OKPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + // Connection phase + case AuthStatusToString(HandshakeV10): + var msg HandshakeV10Packet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthSwitchRequest): + var msg AuthSwitchRequestPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthMoreData): + var msg AuthMoreDataPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthNextFactor): // not supported yet + var msg AuthNextFactorPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + // Command phase + case COM_STMT_PREPARE_OK: + var msg StmtPrepareOkPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case string(Text): + var msg TextResultSet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case string(Binary): + var msg BinaryProtocolResultSet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + default: + return errors.New("failed to unmarshal unknown response packet type") + } + + return nil +} diff --git a/pkg/models/postgres.go b/pkg/models/postgres.go index bb8306b076..faa4da6fdc 100755 --- a/pkg/models/postgres.go +++ b/pkg/models/postgres.go @@ -34,12 +34,12 @@ type Backend struct { CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` Describe pgproto3.Describe `json:"describe,omitempty" yaml:"describe,omitempty"` - Execute pgproto3.Execute `yaml:"-"` + Execute pgproto3.Execute `json:"-" yaml:"-"` Executes []pgproto3.Execute `json:"execute,omitempty" yaml:"execute,omitempty"` Flush pgproto3.Flush `json:"flush,omitempty" yaml:"flush,omitempty"` FunctionCall pgproto3.FunctionCall `json:"function_call,omitempty" yaml:"function_call,omitempty"` GssEncRequest pgproto3.GSSEncRequest `json:"gss_enc_request,omitempty" yaml:"gss_enc_request,omitempty"` - Parse pgproto3.Parse `yaml:"-"` + Parse pgproto3.Parse `json:"-" yaml:"-"` Parses []pgproto3.Parse `json:"parse,omitempty" yaml:"parse,omitempty"` Query pgproto3.Query `json:"query,omitempty" yaml:"query,omitempty"` SSlRequest pgproto3.SSLRequest `json:"ssl_request,omitempty" yaml:"ssl_request,omitempty"` @@ -70,45 +70,33 @@ type Frontend struct { AuthenticationSASLContinue pgproto3.AuthenticationSASLContinue `json:"authentication_sasl_continue,omitempty" yaml:"authentication_sasl_continue,omitempty,flow"` AuthenticationSASLFinal pgproto3.AuthenticationSASLFinal `json:"authentication_sasl_final,omitempty" yaml:"authentication_sasl_final,omitempty,flow"` BackendKeyData pgproto3.BackendKeyData `json:"backend_key_data,omitempty" yaml:"backend_key_data,omitempty"` - BindComplete pgproto3.BindComplete `yaml:"-"` + BindComplete pgproto3.BindComplete `json:"-" yaml:"-"` BindCompletes []pgproto3.BindComplete `json:"bind_complete,omitempty" yaml:"bind_complete,omitempty"` CloseComplete pgproto3.CloseComplete `json:"close_complete,omitempty" yaml:"close_complete,omitempty"` - CommandComplete pgproto3.CommandComplete `yaml:"-"` + CommandComplete pgproto3.CommandComplete `json:"-" yaml:"-"` CommandCompletes []pgproto3.CommandComplete `json:"command_complete,omitempty" yaml:"command_complete,omitempty"` - CopyBothResponse pgproto3.CopyBothResponse `json:"copy_both_response,omitempty" yaml:"copy_both_response,omitempty"` - CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` - CopyInResponse pgproto3.CopyInResponse `json:"copy_in_response,omitempty" yaml:"copy_in_response,omitempty"` - CopyOutResponse pgproto3.CopyOutResponse `json:"copy_out_response,omitempty" yaml:"copy_out_response,omitempty"` - CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` - DataRow pgproto3.DataRow `yaml:"-"` - DataRows []pgproto3.DataRow `json:"data_row,omitempty" yaml:"data_row,omitempty,flow"` - EmptyQueryResponse pgproto3.EmptyQueryResponse `json:"empty_query_response,omitempty" yaml:"empty_query_response,omitempty"` - ErrorResponse pgproto3.ErrorResponse `json:"error_response,omitempty" yaml:"error_response,omitempty"` - FunctionCallResponse pgproto3.FunctionCallResponse `json:"function_call_response,omitempty" yaml:"function_call_response,omitempty"` - NoData pgproto3.NoData `json:"no_data,omitempty" yaml:"no_data,omitempty"` - NoticeResponse pgproto3.NoticeResponse `json:"notice_response,omitempty" yaml:"notice_response,omitempty"` - NotificationResponse pgproto3.NotificationResponse `json:"notification_response,omitempty" yaml:"notification_response,omitempty"` - ParameterDescription pgproto3.ParameterDescription `json:"parameter_description,omitempty" yaml:"parameter_description,omitempty"` - ParameterStatus pgproto3.ParameterStatus `yaml:"-"` - ParameterStatusCombined []pgproto3.ParameterStatus `json:"parameter_status,omitempty" yaml:"parameter_status,omitempty"` - ParseComplete pgproto3.ParseComplete `yaml:"-"` - ParseCompletes []pgproto3.ParseComplete `json:"parse_complete,omitempty" yaml:"parse_complete,omitempty"` - ReadyForQuery pgproto3.ReadyForQuery `json:"ready_for_query,omitempty" yaml:"ready_for_query,omitempty"` - RowDescription pgproto3.RowDescription `json:"row_description,omitempty" yaml:"row_description,omitempty,flow"` - PortalSuspended pgproto3.PortalSuspended `json:"portal_suspended,omitempty" yaml:"portal_suspended,omitempty"` - MsgType byte `json:"msg_type,omitempty" yaml:"msg_type,omitempty"` - AuthType int32 `json:"auth_type" yaml:"auth_type"` - // AuthMechanism string `json:"auth_mechanism,omitempty" yaml:"auth_mechanism,omitempty"` - BodyLen int `json:"body_len,omitempty" yaml:"body_len,omitempty"` -} - -type StartupPacket struct { - Length uint32 - ProtocolVersion uint32 -} - -type RegularPacket struct { - Identifier byte - Length uint32 - Payload []byte + // CopyBothResponse pgproto3.CopyBothResponse `json:"copy_both_response,omitempty" yaml:"copy_both_response,omitempty"` + CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` + // CopyInResponse pgproto3.CopyInResponse `json:"copy_in_response,omitempty" yaml:"copy_in_response,omitempty"` + // CopyOutResponse pgproto3.CopyOutResponse `json:"copy_out_response,omitempty" yaml:"copy_out_response,omitempty"` + CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` + DataRow pgproto3.DataRow `json:"-" yaml:"-"` + DataRows []pgproto3.DataRow `json:"data_row,omitempty" yaml:"data_row,omitempty,flow"` + EmptyQueryResponse pgproto3.EmptyQueryResponse `json:"empty_query_response,omitempty" yaml:"empty_query_response,omitempty"` + ErrorResponse pgproto3.ErrorResponse `json:"error_response,omitempty" yaml:"error_response,omitempty"` + FunctionCallResponse pgproto3.FunctionCallResponse `json:"function_call_response,omitempty" yaml:"function_call_response,omitempty"` + NoData pgproto3.NoData `json:"no_data,omitempty" yaml:"no_data,omitempty"` + NoticeResponse pgproto3.NoticeResponse `json:"notice_response,omitempty" yaml:"notice_response,omitempty"` + NotificationResponse pgproto3.NotificationResponse `json:"notification_response,omitempty" yaml:"notification_response,omitempty"` + ParameterDescription pgproto3.ParameterDescription `json:"parameter_description,omitempty" yaml:"parameter_description,omitempty"` + ParameterStatus pgproto3.ParameterStatus `json:"-" yaml:"-"` + ParameterStatusCombined []pgproto3.ParameterStatus `json:"parameter_status,omitempty" yaml:"parameter_status,omitempty"` + ParseComplete pgproto3.ParseComplete `json:"-" yaml:"-"` + ParseCompletes []pgproto3.ParseComplete `json:"parse_complete,omitempty" yaml:"parse_complete,omitempty"` + ReadyForQuery pgproto3.ReadyForQuery `json:"ready_for_query,omitempty" yaml:"ready_for_query,omitempty"` + RowDescription pgproto3.RowDescription `json:"row_description,omitempty" yaml:"row_description,omitempty,flow"` + PortalSuspended pgproto3.PortalSuspended `json:"portal_suspended,omitempty" yaml:"portal_suspended,omitempty"` + MsgType byte `json:"msg_type,omitempty" yaml:"msg_type,omitempty"` + AuthType int32 `json:"auth_type" yaml:"auth_type"` + BodyLen int `json:"body_len,omitempty" yaml:"body_len,omitempty"` } diff --git a/pkg/platform/docker/docker.go b/pkg/platform/docker/docker.go index 88b1f57a4e..0328e9b00c 100644 --- a/pkg/platform/docker/docker.go +++ b/pkg/platform/docker/docker.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types" dockerContainerPkg "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" @@ -154,7 +153,7 @@ func (idc *Impl) StopAndRemoveDockerContainer() error { } } - removeOptions := types.ContainerRemoveOptions{ + removeOptions := dockerContainerPkg.RemoveOptions{ RemoveVolumes: true, Force: true, } @@ -175,7 +174,7 @@ func (idc *Impl) NetworkExists(networkName string) (bool, error) { defer cancel() // Retrieve all networks. - networks, err := idc.NetworkList(ctx, types.NetworkListOptions{}) + networks, err := idc.NetworkList(ctx, network.ListOptions{}) if err != nil { return false, fmt.Errorf("error retrieving networks: %v", err) } @@ -195,7 +194,7 @@ func (idc *Impl) CreateNetwork(networkName string) error { ctx, cancel := context.WithTimeout(context.Background(), idc.timeoutForDockerQuery) defer cancel() - _, err := idc.NetworkCreate(ctx, networkName, types.NetworkCreate{ + _, err := idc.NetworkCreate(ctx, networkName, network.CreateOptions{ Driver: "bridge", }) @@ -360,7 +359,7 @@ func (idc *Impl) GetHostWorkingDirectory() (string, error) { // Loop through container mounts and find the mount for current directory in the container for _, mount := range containerMounts { if mount.Destination == curDir { - idc.logger.Debug(fmt.Sprintf("found mount for %s in keploy-v2 container", curDir), zap.Any("mount", mount)) + idc.logger.Info(fmt.Sprintf("found mount for %s in keploy-v2 container", curDir), zap.Any("mount", mount)) return mount.Source, nil } } @@ -522,6 +521,42 @@ func (idc *Impl) SetKeployNetwork(c *Compose) (*NetworkInfo, error) { return networkInfo, nil } +func (idc *Impl) SetInitPid(c *Compose, containerName string) error { + for _, service := range c.Services.Content { + var containerNameMatch bool + var pidFound bool + + for i := 0; i < len(service.Content)-1; i++ { + if service.Content[i].Kind == yaml.ScalarNode && service.Content[i].Value == "container_name" && + service.Content[i+1].Kind == yaml.ScalarNode && service.Content[i+1].Value == containerName { + containerNameMatch = true + break + } + } + + if containerNameMatch { + for _, item := range service.Content { + if item.Value == "pid" { + pidFound = true + break + } + } + + // Add `pid: container:keploy-init` only if not already present + if !pidFound { + service.Content = append(service.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pid"}, + &yaml.Node{ + Kind: yaml.ScalarNode, + Value: "container:keploy-init", + }, + ) + } + } + } + return nil +} + // IsContainerRunning check if the container is already running or not, required for docker start command. func (idc *Impl) IsContainerRunning(containerName string) (bool, error) { diff --git a/pkg/platform/docker/service.go b/pkg/platform/docker/service.go index d41dea9263..7e22d13a4f 100644 --- a/pkg/platform/docker/service.go +++ b/pkg/platform/docker/service.go @@ -27,7 +27,7 @@ type Client interface { SetKeployNetwork(c *Compose) (*NetworkInfo, error) ReadComposeFile(filePath string) (*Compose, error) WriteComposeFile(compose *Compose, path string) error - + SetInitPid(c *Compose, containerName string) error IsContainerRunning(containerName string) (bool, error) CreateVolume(ctx context.Context, volumeName string, recreate bool) error } diff --git a/pkg/platform/docker/util.go b/pkg/platform/docker/util.go index f35f46ef73..0827154284 100644 --- a/pkg/platform/docker/util.go +++ b/pkg/platform/docker/util.go @@ -1,14 +1,34 @@ -//go:build linux +//go:build !windows package docker import ( + "context" + "errors" "fmt" + "os" + "os/exec" "regexp" + "runtime" + "strings" + "syscall" + "github.com/docker/docker/api/types/network" + "go.keploy.io/server/v2/config" "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/term" ) +type ConfigStruct struct { + DockerImage string + Envs map[string]string +} + +var DockerConfig = ConfigStruct{ + DockerImage: "ghcr.io/keploy/keploy", +} + func ParseDockerCmd(cmd string, kind utils.CmdType, idc Client) (string, string, error) { // Regular expression patterns @@ -51,3 +71,254 @@ func ParseDockerCmd(cmd string, kind utils.CmdType, idc Client) (string, string, return containerName, networkName, nil } + +func GenerateDockerEnvs(config ConfigStruct) string { + var envs []string + for key, value := range config.Envs { + envs = append(envs, fmt.Sprintf("-e %s='%s'", key, value)) + } + return strings.Join(envs, " ") +} + +// StartInDocker will check if the docker command is provided as an input +// then start the Keploy as a docker container and run the command +// should also return a boolean if the execution is moved to docker +func StartInDocker(ctx context.Context, logger *zap.Logger, conf *config.Config) error { + + if DockerConfig.Envs == nil { + DockerConfig.Envs = map[string]string{ + "INSTALLATION_ID": conf.InstallationID, + } + } else { + DockerConfig.Envs["INSTALLATION_ID"] = conf.InstallationID + } + + err := RunInDocker(ctx, logger) + if err != nil { + utils.LogError(logger, err, "failed to run the command in docker") + return err + } + // gracefully exit the current process + logger.Info("exiting the current process as the command is moved to docker") + os.Exit(0) + return nil +} + +func RunInDocker(ctx context.Context, logger *zap.Logger) error { + //Get the correct keploy alias. + keployAlias, err := getAlias(ctx, logger) + if err != nil { + return err + } + + client, err := New(logger) + if err != nil { + utils.LogError(logger, err, "failed to initalise docker") + return err + } + + addKeployNetwork(ctx, logger, client) + err = client.CreateVolume(ctx, "debugfs", true) + if err != nil { + utils.LogError(logger, err, "failed to debugfs volume") + return err + } + + var cmd *exec.Cmd + + // Detect the operating system + if runtime.GOOS == "windows" { + var args []string + args = append(args, "/C") + args = append(args, strings.Split(keployAlias, " ")...) + args = append(args, os.Args[1:]...) + // Use cmd.exe /C for Windows + cmd = exec.CommandContext( + ctx, + "cmd.exe", + args..., + ) + } else { + // Use sh -c for Unix-like systems + cmd = exec.CommandContext( + ctx, + "sh", + "-c", + keployAlias, + ) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + } + } + + cmd.Cancel = func() error { + err := utils.SendSignal(logger, -cmd.Process.Pid, syscall.SIGINT) + if err != nil { + utils.LogError(logger, err, "failed to start stop docker") + return err + } + return nil + } + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logger.Info("running the following command in docker", zap.String("command", cmd.String())) + err = cmd.Run() + if err != nil { + if ctx.Err() == context.Canceled { + return ctx.Err() + } + utils.LogError(logger, err, "failed to start keploy in docker") + return err + } + return nil +} + +func getAlias(ctx context.Context, logger *zap.Logger) (string, error) { + // Get the name of the operating system. + osName := runtime.GOOS + //TODO: configure the hardcoded port mapping + img := DockerConfig.DockerImage + ":v" + utils.Version + logger.Info("Starting keploy in docker with image", zap.String("image:", img)) + envs := GenerateDockerEnvs(DockerConfig) + if envs != "" { + envs = envs + " " + } + var ttyFlag string + + if term.IsTerminal(int(os.Stdin.Fd())) { + // ttyFlag = " -it " + ttyFlag = " " + } else { + ttyFlag = " " + } + + switch osName { + case "linux": + alias := "sudo docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + " -v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + case "windows": + // Get the current working directory + pwd, err := os.Getwd() + if err != nil { + utils.LogError(logger, err, "failed to get the current working directory") + } + dpwd := convertPathToUnixStyle(pwd) + cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") + out, err := cmd.Output() + if err != nil { + utils.LogError(logger, err, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] + if len(dockerContext) == 0 { + utils.LogError(logger, nil, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext = strings.Split(dockerContext, "\n")[0] + if dockerContext == "colima" { + logger.Info("Starting keploy in docker with colima context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img + return alias, nil + } + // if default docker context is used + logger.Info("Starting keploy in docker with default context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img + return alias, nil + case "darwin": + cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") + out, err := cmd.Output() + if err != nil { + utils.LogError(logger, err, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] + if len(dockerContext) == 0 { + utils.LogError(logger, nil, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext = strings.Split(dockerContext, "\n")[0] + if dockerContext == "colima" { + logger.Info("Starting keploy in docker with colima context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + } + // if default docker context is used + logger.Info("Starting keploy in docker with default context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + } + return "", errors.New("failed to get alias") +} + +func addKeployNetwork(ctx context.Context, logger *zap.Logger, client Client) { + + // Check if the 'keploy-network' network exists + networks, err := client.NetworkList(ctx, network.ListOptions{}) + if err != nil { + logger.Debug("failed to list docker networks") + return + } + + for _, network := range networks { + if network.Name == "keploy-network" { + logger.Debug("keploy network already exists") + return + } + } + + // Create the 'keploy' network if it doesn't exist + _, err = client.NetworkCreate(ctx, "keploy-network", network.CreateOptions{}) + if err != nil { + logger.Debug("failed to create keploy network") + return + } + + logger.Debug("keploy network created") +} + +func convertPathToUnixStyle(path string) string { + // Replace backslashes with forward slashes + unixPath := strings.Replace(path, "\\", "/", -1) + // Remove 'C:' + if len(unixPath) > 1 && unixPath[1] == ':' { + unixPath = unixPath[2:] + } + return unixPath +} + +// ExtractPidNamespaceInode extracts the inode of the PID namespace of a given PID +func ExtractPidNamespaceInode(pid int) (string, error) { + // Check the OS + if runtime.GOOS != "linux" { + // Execute command in the container to get the PID namespace + output, err := exec.Command("docker", "exec", "keploy-init", "stat", "/proc/1/ns/pid").Output() + if err != nil { + return "", err + } + outputStr := string(output) + + // Use a regular expression to extract the inode from the output + re := regexp.MustCompile(`pid:\[(\d+)\]`) + match := re.FindStringSubmatch(outputStr) + + if len(match) < 2 { + return "", fmt.Errorf("failed to extract PID namespace inode") + } + + pidNamespace := match[1] + return pidNamespace, nil + } + + // Check the namespace file in /proc + nsPath := fmt.Sprintf("/proc/%d/ns/pid", pid) + fileInfo, err := os.Stat(nsPath) + if err != nil { + return "", err + } + + // Retrieve inode number + inode := fileInfo.Sys().(*syscall.Stat_t).Ino + return fmt.Sprintf("%d", inode), nil +} diff --git a/pkg/platform/http/agent.go b/pkg/platform/http/agent.go new file mode 100644 index 0000000000..6ab3e4d479 --- /dev/null +++ b/pkg/platform/http/agent.go @@ -0,0 +1,761 @@ +//go:build !windows + +// Package http contains the client side code to communicate with the agent server +package http + +import ( + "bytes" + "context" + _ "embed" // necessary for embedding + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "sync" + "syscall" + "time" + + "github.com/docker/docker/api/types/events" + "go.keploy.io/server/v2/config" + "go.keploy.io/server/v2/pkg/agent/hooks" + "go.keploy.io/server/v2/pkg/client/app" + "go.keploy.io/server/v2/pkg/models" + kdocker "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type AgentClient struct { + logger *zap.Logger + dockerClient kdocker.Client //embedding the docker client to transfer the docker client methods to the core object + apps sync.Map + client http.Client + conf *config.Config +} + +//go:embed assets/initStop.sh +var initStopScript []byte + +func New(logger *zap.Logger, client kdocker.Client, c *config.Config) *AgentClient { + + return &AgentClient{ + logger: logger, + dockerClient: client, + client: http.Client{}, + conf: c, + } +} + +func (a *AgentClient) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + requestBody := models.IncomingReq{ + IncomingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for incoming request") + return nil, fmt.Errorf("error marshaling request body for incoming request: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/incoming", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for incoming request") + return nil, fmt.Errorf("error creating request for incoming request: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get incoming: %s", err.Error()) + } + + // Ensure response body is closed when we're done + go func() { + <-ctx.Done() + if res.Body != nil { + _ = res.Body.Close() + } + }() + + // Create a channel to stream TestCase data + tcChan := make(chan *models.TestCase) + + go func() { + defer close(tcChan) + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for incoming request") + } + }() + + decoder := json.NewDecoder(res.Body) + + for { + var testCase models.TestCase + if err := decoder.Decode(&testCase); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + // End of the stream + break + } + utils.LogError(a.logger, err, "failed to decode test case from stream") + break + } + + select { + case <-ctx.Done(): + // If the context is done, exit the loop + return + case tcChan <- &testCase: + // fmt.Println("Test case received for client", id, "TESTCASE", testCase) + // Send the decoded test case to the channel + } + } + }() + + return tcChan, nil +} + +func (a *AgentClient) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { + requestBody := models.OutgoingReq{ + OutgoingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for mock outgoing") + return nil, fmt.Errorf("error marshaling request body for mock outgoing: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/outgoing", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for mock outgoing") + return nil, fmt.Errorf("error creating request for mock outgoing: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get outgoing response: %s", err.Error()) + } + + // Create a channel to stream Mock data + mockChan := make(chan *models.Mock) + + // use error group instead of go routine + grp, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return nil, fmt.Errorf("failed to get errorgroup from the context") + } + + grp.Go(func() error { + defer close(mockChan) + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for getoutgoing") + } + }() + + decoder := json.NewDecoder(res.Body) + + for { + var mock models.Mock + if err := decoder.Decode(&mock); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + // End of the stream + break + } + utils.LogError(a.logger, err, "failed to decode mock from stream") + break + } + + select { + case <-ctx.Done(): + // If the context is done, exit the loop + return nil + case mockChan <- &mock: + // Send the decoded mock to the channel + } + } + return nil + }) + + return mockChan, nil +} + +func (a *AgentClient) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { + // make a request to the server to mock outgoing + requestBody := models.OutgoingReq{ + OutgoingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for mock outgoing") + return fmt.Errorf("error marshaling request body for mock outgoing: %s", err.Error()) + } + + // mock outgoing request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/mock", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for mock outgoing") + return fmt.Errorf("error creating request for mock outgoing: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request for mockOutgoing: %s", err.Error()) + } + + var mockResp models.AgentResp + err = json.NewDecoder(res.Body).Decode(&mockResp) + if err != nil { + return fmt.Errorf("failed to decode response body for mock outgoing: %s", err.Error()) + } + + if mockResp.Error != nil { + return mockResp.Error + } + + return nil + +} + +func (a *AgentClient) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { + requestBody := models.SetMocksReq{ + Filtered: filtered, + UnFiltered: unFiltered, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for setmocks") + return fmt.Errorf("error marshaling request body for setmocks: %s", err.Error()) + } + + // mock outgoing request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/setmocks", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for setmocks outgoing") + return fmt.Errorf("error creating request for set mocks: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request for setmocks: %s", err.Error()) + } + + var mockResp models.AgentResp + err = json.NewDecoder(res.Body).Decode(&mockResp) + if err != nil { + return fmt.Errorf("failed to decode response body for setmocks: %s", err.Error()) + } + + if mockResp.Error != nil { + return mockResp.Error + } + + return nil +} + +func (a *AgentClient) GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) { + // Create the URL with query parameters + url := fmt.Sprintf("http://localhost:%d/agent/consumedmocks?id=%d", a.conf.Agent.Port, id) + + // Create a new GET request with the query parameter + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %s", err.Error()) + } + + req.Header.Set("Content-Type", "application/json") + + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request for mockOutgoing: %s", err.Error()) + } + + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for getconsumedmocks") + } + }() + + var consumedMocks []string + err = json.NewDecoder(res.Body).Decode(&consumedMocks) + if err != nil { + return nil, fmt.Errorf("failed to decode response body: %s", err.Error()) + } + + return consumedMocks, nil +} + +func (a *AgentClient) GetContainerIP(_ context.Context, clientID uint64) (string, error) { + + app, err := a.getApp(clientID) + if err != nil { + utils.LogError(a.logger, err, "failed to get app") + return "", err + } + + ip := app.ContainerIPv4Addr() + a.logger.Debug("ip address of the target app container", zap.Any("ip", ip)) + if ip == "" { + return "", fmt.Errorf("failed to get the IP address of the app container. Try increasing --delay (in seconds)") + } + + return ip, nil +} + +func (a *AgentClient) Run(ctx context.Context, clientID uint64, _ models.RunOptions) models.AppError { + + app, err := a.getApp(clientID) + if err != nil { + utils.LogError(a.logger, err, "failed to get app while running") + return models.AppError{AppErrorType: models.ErrInternal, Err: err} + } + + runAppErrGrp, runAppCtx := errgroup.WithContext(ctx) + + appErrCh := make(chan models.AppError, 1) + + defer func() { + err := runAppErrGrp.Wait() + + if err != nil { + utils.LogError(a.logger, err, "failed to stop the app") + } + }() + + runAppErrGrp.Go(func() error { + defer utils.Recover(a.logger) + defer close(appErrCh) + appErr := app.Run(runAppCtx) + if appErr.Err != nil { + utils.LogError(a.logger, appErr.Err, "error while running the app") + appErrCh <- appErr + } + return nil + }) + + select { + case <-runAppCtx.Done(): + return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: nil} + case appErr := <-appErrCh: + return appErr + } +} + +func (a *AgentClient) Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) { + + clientID := utils.GenerateID() + isDockerCmd := utils.IsDockerCmd(utils.CmdType(opts.CommandType)) + + // check if the agent is running + isAgentRunning := a.isAgentRunning(ctx) + if opts.EnableTesting { + fmt.Println("Testing is enabled") + isAgentRunning = false + } + if !isAgentRunning { + // Start the keploy agent as a detached process and pipe the logs into a file + if !isDockerCmd && runtime.GOOS != "linux" { + return 0, fmt.Errorf("Operating system not supported for this feature") + } + + if isDockerCmd { + // run the docker container instead of the agent binary + go func() { + if err := a.StartInDocker(ctx, a.logger); err != nil && !errors.Is(ctx.Err(), context.Canceled) { + a.logger.Error("failed to start Docker agent", zap.Error(err)) + } + }() + } else { + // Open the log file in append mode or create it if it doesn't exist + logFile, err := os.OpenFile("keploy_agent.log", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + utils.LogError(a.logger, err, "failed to open log file") + return 0, err + } + + defer func() { + err := logFile.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close agent log file") + } + }() + + keployBin, err := utils.GetCurrentBinaryPath() + + if err != nil { + utils.LogError(a.logger, err, "failed to get current keploy binary path") + return 0, err + } + agentCmd := exec.Command("sudo", keployBin, "agent", "--port", strconv.Itoa(int(a.conf.ServerPort)), "--proxy-port", strconv.Itoa(int(a.conf.ProxyPort))) + agentCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} // Detach the process + + // Redirect the standard output and error to the log file + agentCmd.Stdout = logFile + agentCmd.Stderr = logFile + + if err := agentCmd.Start(); err != nil { + utils.LogError(a.logger, err, "failed to start keploy agent") + return 0, err + } + a.logger.Info("keploy agent started", zap.Int("pid", agentCmd.Process.Pid)) + } + } + + // Channel to monitor if the agent is up and running + runningChan := make(chan bool) + + go func() { + for { + select { + case <-ctx.Done(): + // If the context is canceled, close the channel and return immediately + close(runningChan) + return + default: + if a.isAgentRunning(ctx) { + runningChan <- true + return + } + time.Sleep(1 * time.Second) // Poll every second + } + } + }() + + // Wait until the agent is ready or context is canceled + select { + case <-ctx.Done(): + // Handle context cancellation gracefully if Ctrl+C is pressed + a.logger.Info("Setup was canceled before the agent became ready") + return 0, fmt.Errorf("setup canceled before agent startup") + case <-runningChan: + // Proceed with setup if the agent becomes ready + a.logger.Info("Agent is now running, proceeding with setup") + } + + // Continue with app setup and registration as per normal flow + usrApp := app.NewApp(a.logger, clientID, cmd, a.dockerClient, app.Options{ + DockerNetwork: opts.DockerNetwork, + Container: opts.Container, + DockerDelay: opts.DockerDelay, + }) + a.apps.Store(clientID, usrApp) + + err := usrApp.Setup(ctx) + if err != nil { + utils.LogError(a.logger, err, "failed to setup app") + return 0, err + } + if isDockerCmd { + opts.DockerNetwork = usrApp.KeployNetwork + inode, err := a.Initcontainer(ctx, app.Options{ + DockerNetwork: opts.DockerNetwork, + Container: opts.Container, + DockerDelay: opts.DockerDelay, + }) + if err != nil { + utils.LogError(a.logger, err, "failed to setup init container") + return 0, err + } + opts.AppInode = inode + } + + opts.ClientID = clientID + if err := a.RegisterClient(ctx, opts); err != nil { + utils.LogError(a.logger, err, "failed to register client") + return 0, err + } + isAgentRunning = a.isAgentRunning(ctx) + if !isAgentRunning { + return 0, fmt.Errorf("keploy agent is not running, please start the agent first") + } + + return clientID, nil +} + +func (a *AgentClient) getApp(clientID uint64) (*app.App, error) { + ap, ok := a.apps.Load(clientID) + if !ok { + return nil, fmt.Errorf("app with id:%v not found", clientID) + } + + // type assertion on the app + h, ok := ap.(*app.App) + if !ok { + return nil, fmt.Errorf("failed to type assert app with id:%v", clientID) + } + + return h, nil +} + +// RegisterClient registers the client with the server +func (a *AgentClient) RegisterClient(ctx context.Context, opts models.SetupOptions) error { + + isAgent := a.isAgentRunning(ctx) + if !isAgent { + return fmt.Errorf("keploy agent is not running, please start the agent first") + } + + // Register the client with the server + clientPid := uint32(os.Getpid()) + + // start the app container and get the inode number + // keploy agent would have already runnning, + var inode uint64 + var err error + if runtime.GOOS == "linux" { + // send the network info to the kernel + inode, err = hooks.GetSelfInodeNumber() + if err != nil { + a.logger.Error("failed to get inode number") + } + } + + // Register the client with the server + requestBody := models.RegisterReq{ + SetupOptions: models.SetupOptions{ + DockerNetwork: opts.DockerNetwork, + ClientNsPid: clientPid, + Mode: opts.Mode, + ClientID: opts.ClientID, + ClientInode: inode, + IsDocker: a.conf.Agent.IsDocker, + AppInode: opts.AppInode, + ProxyPort: a.conf.ProxyPort, + }, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for register client") + return fmt.Errorf("error marshaling request body for register client: %s", err.Error()) + } + + resp, err := a.client.Post(fmt.Sprintf("http://localhost:%d/agent/register", a.conf.Agent.Port), "application/json", bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to send register client request") + return fmt.Errorf("error sending register client request: %s", err.Error()) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to register client: %s", resp.Status) + } + + a.logger.Info("Client registered successfully with clientId", zap.Uint64("clientID", opts.ClientID)) + + // TODO: Read the response body in which we return the app id + var RegisterResp models.AgentResp + err = json.NewDecoder(resp.Body).Decode(&RegisterResp) + if err != nil { + utils.LogError(a.logger, err, "failed to decode response body for register client") + return fmt.Errorf("error decoding response body for register client: %s", err.Error()) + } + + if RegisterResp.Error != nil { + return RegisterResp.Error + } + + return nil +} + +func (a *AgentClient) UnregisterClient(ctx context.Context, unregister models.UnregisterReq) error { + // Unregister the client with the server + isAgentRunning := a.isAgentRunning(context.Background()) + if !isAgentRunning { + a.logger.Warn("keploy agent is not running, skipping unregister client") + return io.EOF + } + + // Passed background context as we dont want to cancel the unregister request upon client ctx cancellation + fmt.Println("Unregistering the client with the server") + requestJSON, err := json.Marshal(unregister) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for unregister client") + return fmt.Errorf("error marshaling request body for unregister client: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(context.Background(), "POST", fmt.Sprintf("http://localhost:%d/agent/unregister", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for unregister client") + return fmt.Errorf("error creating request for unregister client: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := a.client.Do(req) + if err != nil && err != io.EOF { + return fmt.Errorf("failed to send request for unregister client: %s", err.Error()) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to unregister client: %s", resp.Status) + } + + return nil +} + +func (a *AgentClient) StartInDocker(ctx context.Context, logger *zap.Logger) error { + // Start the Keploy agent in a Docker container, directly using the passed context for cancellation + agentCtx := context.WithoutCancel(ctx) + err := kdocker.StartInDocker(agentCtx, logger, &config.Config{ + InstallationID: a.conf.InstallationID, + }) + if err != nil { + utils.LogError(logger, err, "failed to start keploy agent in docker") + return err + } + return nil +} + +func (a *AgentClient) Initcontainer(ctx context.Context, opts app.Options) (uint64, error) { + // Create a temporary file for the embedded initStop.sh script + initFile, err := os.CreateTemp("", "initStop.sh") + if err != nil { + a.logger.Error("failed to create temporary file", zap.Error(err)) + return 0, err + } + defer func() { + err := os.Remove(initFile.Name()) + if err != nil { + a.logger.Error("failed to remove temporary file", zap.Error(err)) + } + }() + + _, err = initFile.Write(initStopScript) + if err != nil { + a.logger.Error("failed to write script to temporary file", zap.Error(err)) + return 0, err + } + + // Close the file after writing to avoid 'text file busy' error + if err := initFile.Close(); err != nil { + a.logger.Error("failed to close temporary file", zap.Error(err)) + return 0, err + } + + err = os.Chmod(initFile.Name(), 0755) + if err != nil { + a.logger.Error("failed to make temporary script executable", zap.Error(err)) + return 0, err + } + + // Create a channel to signal when the container starts + containerStarted := make(chan struct{}) + + // Start the Docker events listener in a separate goroutine + go func() { + events, errs := a.dockerClient.Events(ctx, events.ListOptions{}) + for { + select { + case event := <-events: + if event.Type == "container" && event.Action == "start" && event.Actor.Attributes["name"] == "keploy-init" { + a.logger.Info("Container keploy-init started") + containerStarted <- struct{}{} + return + } + case err := <-errs: + a.logger.Error("Error while listening to Docker events", zap.Error(err)) + return + case <-ctx.Done(): + return + } + } + }() + + // Start the init container to get the PID namespace inode + cmdCancel := func(cmd *exec.Cmd) func() error { + return func() error { + a.logger.Info("sending SIGINT to the Initcontainer", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) + err := utils.SendSignal(a.logger, -cmd.Process.Pid, syscall.SIGINT) + return err + } + } + + cmd := fmt.Sprintf("docker run --network=%s --name keploy-init --rm -v%s:/initStop.sh alpine /initStop.sh", opts.DockerNetwork, initFile.Name()) + + // Execute the command + grp, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return 0, fmt.Errorf("failed to get errorgroup from the context") + } + + grp.Go(func() error { + println("Executing the init container command") + cmdErr := utils.ExecuteCommand(ctx, a.logger, cmd, cmdCancel, 25*time.Second) + if cmdErr.Err != nil && cmdErr.Type == utils.Init { + utils.LogError(a.logger, cmdErr.Err, "failed to execute init container command") + } + + println("Init container stopped") + return nil + }) + + // Wait for the container to start or context to cancel + select { + case <-containerStarted: + a.logger.Info("keploy-init container is running") + case <-ctx.Done(): + return 0, fmt.Errorf("context canceled while waiting for container to start") + } + + // Get the PID of the container's first process + inspect, err := a.dockerClient.ContainerInspect(ctx, "keploy-init") + if err != nil { + a.logger.Error("failed to inspect container", zap.Error(err)) + return 0, err + } + + pid := inspect.State.Pid + a.logger.Info("Container PID", zap.Int("pid", pid)) + + // Extract inode from the PID namespace + pidNamespaceInode, err := kdocker.ExtractPidNamespaceInode(pid) + if err != nil { + a.logger.Error("failed to extract PID namespace inode", zap.Error(err)) + return 0, err + } + + a.logger.Info("PID Namespace Inode", zap.String("inode", pidNamespaceInode)) + iNode, err := strconv.ParseUint(pidNamespaceInode, 10, 64) + if err != nil { + a.logger.Error("failed to convert inode to uint64", zap.Error(err)) + return 0, err + } + return iNode, nil +} + +func (a *AgentClient) isAgentRunning(ctx context.Context) bool { + + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://localhost:%d/agent/health", a.conf.Agent.Port), nil) + if err != nil { + utils.LogError(a.logger, err, "failed to send request to the agent server") + } + + resp, err := a.client.Do(req) + if err != nil { + a.logger.Info("Keploy agent is not running in background, starting the agent") + return false + } + a.logger.Info("Setup request sent to the server", zap.String("status", resp.Status)) + return true +} diff --git a/pkg/platform/http/assets/initStop.sh b/pkg/platform/http/assets/initStop.sh new file mode 100755 index 0000000000..87bb6bd30a --- /dev/null +++ b/pkg/platform/http/assets/initStop.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Handle SIGINT and SIGTERM signals, forwarding them to the sleep process +trap 'echo "Init Container received SIGTERM or SIGINT, exiting..."; exit' SIGINT SIGTERM + +# Start sleep infinity to keep the container running in the background +sleep infinity & + +# Wait for the background process and forward any signals +wait $! diff --git a/pkg/platform/yaml/configdb/user/db.go b/pkg/platform/yaml/configdb/user/db.go index 5dc3fae5fa..33b2091221 100644 --- a/pkg/platform/yaml/configdb/user/db.go +++ b/pkg/platform/yaml/configdb/user/db.go @@ -89,16 +89,13 @@ func New(logger *zap.Logger, cfg *config.Config) *Db { func (db *Db) GetInstallationID(_ context.Context) (string, error) { var id string var err error - inDocker := os.Getenv("KEPLOY_INDOCKER") - if inDocker == "true" { - id = os.Getenv("INSTALLATION_ID") - } else { - id, err = machineid.ID() - if err != nil { - db.logger.Debug("failed to get machine id", zap.Error(err)) - return "", nil - } + + id, err = machineid.ID() + if err != nil { + db.logger.Debug("failed to get machine id", zap.Error(err)) + return "", nil } + if id == "" { db.logger.Debug("got empty machine id") return "", nil diff --git a/pkg/platform/yaml/mockdb/util.go b/pkg/platform/yaml/mockdb/util.go index 31b3187c84..025e378c69 100644 --- a/pkg/platform/yaml/mockdb/util.go +++ b/pkg/platform/yaml/mockdb/util.go @@ -174,6 +174,7 @@ func EncodeMock(mock *models.Mock, logger *zap.Logger) (*yaml.NetworkTrafficDoc, return nil, err } default: + utils.LogError(logger, nil, "failed to marshal the recorded mock into yaml due to invalid kind of mock") return nil, errors.New("type of mock is invalid") } diff --git a/pkg/platform/yaml/utils.go b/pkg/platform/yaml/utils.go index 52367e57cc..3c70695c3b 100755 --- a/pkg/platform/yaml/utils.go +++ b/pkg/platform/yaml/utils.go @@ -171,7 +171,6 @@ func NewSessionIndex(path string, Logger *zap.Logger) (string, error) { } for _, v := range files { - // fmt.Println("name for the file", v.Name()) fileName := filepath.Base(v.Name()) fileNamePackets := strings.Split(fileName, "-") if len(fileNamePackets) == 3 { diff --git a/pkg/service/agent/agent.go b/pkg/service/agent/agent.go new file mode 100644 index 0000000000..60e4478153 --- /dev/null +++ b/pkg/service/agent/agent.go @@ -0,0 +1,299 @@ +//go:build linux + +// Package agent contains methods for setting up hooks and proxy along with registering keploy clients. +package agent + +import ( + "context" + "errors" + "fmt" + + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" + "go.keploy.io/server/v2/pkg/models" + kdocker "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type Agent struct { + logger *zap.Logger + agent.Proxy // embedding the Proxy interface to transfer the proxy methods to the core object + agent.Hooks // embedding the Hooks interface to transfer the hooks methods to the core object + agent.Tester // embedding the Tester interface to transfer the tester methods to the core object + dockerClient kdocker.Client //embedding the docker client to transfer the docker client methods to the core object + proxyStarted bool +} + +func New(logger *zap.Logger, hook agent.Hooks, proxy agent.Proxy, tester agent.Tester, client kdocker.Client) *Agent { + return &Agent{ + logger: logger, + Hooks: hook, + Proxy: proxy, + Tester: tester, + dockerClient: client, + } +} + +// Setup will create a new app and store it in the map, all the setup will be done here +func (a *Agent) Setup(ctx context.Context, opts models.SetupOptions) error { + + a.logger.Info("Starting the agent in ", zap.String(string(opts.Mode), "mode")) + err := a.Hook(ctx, 0, models.HookOptions{ + Mode: opts.Mode, + IsDocker: opts.IsDocker, + EnableTesting: opts.EnableTesting, + }) + if err != nil { + a.logger.Error("failed to hook into the app", zap.Error(err)) + } + + <-ctx.Done() + a.logger.Info("Context cancelled, stopping the agent") + return context.Canceled + +} + +func (a *Agent) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + return a.Hooks.Record(ctx, id, opts) +} + +func (a *Agent) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { + m := make(chan *models.Mock, 500) + + err := a.Proxy.Record(ctx, id, m, opts) + if err != nil { + return nil, err + } + + return m, nil +} + +func (a *Agent) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { + a.logger.Debug("Inside MockOutgoing of agent binary !!") + + err := a.Proxy.Mock(ctx, id, opts) + if err != nil { + return err + } + + return nil +} + +func (a *Agent) Hook(ctx context.Context, id uint64, opts models.HookOptions) error { + hookErr := errors.New("failed to hook into the app") + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // create a new error group for the hooks + hookErrGrp, _ := errgroup.WithContext(ctx) + hookCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the hookCtx to control the lifecycle of the hooks + hookCtx, hookCtxCancel := context.WithCancel(hookCtx) + hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) + + // create a new error group for the proxy + proxyErrGrp, _ := errgroup.WithContext(ctx) + proxyCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the proxyCtx to control the lifecycle of the proxy + proxyCtx, proxyCtxCancel := context.WithCancel(proxyCtx) + proxyCtx = context.WithValue(proxyCtx, models.ErrGroupKey, proxyErrGrp) + + hookErrGrp.Go(func() error { + <-ctx.Done() + + proxyCtxCancel() + err := proxyErrGrp.Wait() + if err != nil { + utils.LogError(a.logger, err, "failed to stop the proxy") + } + + hookCtxCancel() + err = hookErrGrp.Wait() + if err != nil { + utils.LogError(a.logger, err, "failed to unload the hooks") + } + return nil + }) + + // load hooks if the mode changes .. + err := a.Hooks.Load(hookCtx, id, agent.HookCfg{ + ClientID: id, + Pid: 0, + IsDocker: opts.IsDocker, + KeployIPV4: "172.18.0.2", + Mode: opts.Mode, + }) + + if err != nil { + utils.LogError(a.logger, err, "failed to load hooks") + return hookErr + } + + if a.proxyStarted { + a.logger.Info("Proxy already started") + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + err = a.Proxy.StartProxy(proxyCtx, agent.ProxyOptions{ + DNSIPv4Addr: "172.18.0.2", + //DnsIPv6Addr: "" + }) + if err != nil { + utils.LogError(a.logger, err, "failed to start proxy") + return hookErr + } + + a.proxyStarted = true + + // For keploy test bench + // Doubt: if this is enabled automatically + fmt.Println("opts.EnableTesting", opts.EnableTesting) + opts.EnableTesting = true + if opts.EnableTesting { + + // enable testing in the app + // a.EnableTesting = true + // a.Mode = opts.Mode + + // Setting up the test bench + err := a.Tester.Setup(ctx, models.TestingOptions{Mode: opts.Mode}) + if err != nil { + utils.LogError(a.logger, err, "error while setting up the test bench environment") + return errors.New("failed to setup the test bench") + } + } + + return nil +} + +func (a *Agent) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { + a.logger.Debug("Inside SetMocks of agent binary !!") + return a.Proxy.SetMocks(ctx, id, filtered, unFiltered) +} + +func (a *Agent) GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) { + return a.Proxy.GetConsumedMocks(ctx, id) +} + +func (a *Agent) DeRegisterClient(ctx context.Context, unregister models.UnregisterReq) error { + fmt.Println("Inside DeRegisterClient of agent binary !!") + // send the info of the mode if its test mode we dont need to send the last mock + + if unregister.Mode != models.MODE_TEST { + err := a.Proxy.MakeClientDeRegisterd(ctx) + if err != nil { + return err + } + } + err := a.Hooks.DeleteKeployClientInfo(unregister.ClientID) + if err != nil { + return err + } + + return nil +} + +func (a *Agent) RegisterClient(ctx context.Context, opts models.SetupOptions) error { + + a.logger.Info("Registering the client with the keploy server") + // send the network info to the kernel + err := a.SendNetworkInfo(ctx, opts) + if err != nil { + a.logger.Error("failed to send network info to the kernel", zap.Error(err)) + return err + } + + clientInfo := structs.ClientInfo{ + KeployClientNsPid: opts.ClientNsPid, + IsDockerApp: 0, + KeployClientInode: opts.ClientInode, + AppInode: opts.AppInode, + } + + switch opts.Mode { + case models.MODE_RECORD: + clientInfo.Mode = uint32(1) + case models.MODE_TEST: + clientInfo.Mode = uint32(2) + default: + clientInfo.Mode = uint32(0) + } + + if opts.IsDocker { + clientInfo.IsDockerApp = 1 + } + + return a.Hooks.SendKeployClientInfo(opts.ClientID, clientInfo) +} + +func (a *Agent) SendNetworkInfo(ctx context.Context, opts models.SetupOptions) error { + if !opts.IsDocker { + proxyIP, err := hooks.IPv4ToUint32("127.0.0.1") + if err != nil { + return err + } + proxyInfo := structs.ProxyInfo{ + IP4: proxyIP, + IP6: [4]uint32{0, 0, 0, 0}, + Port: opts.ProxyPort, + } + err = a.Hooks.SendClientProxyInfo(opts.ClientID, proxyInfo) + if err != nil { + return err + } + return nil + } + + inspect, err := a.dockerClient.ContainerInspect(ctx, "keploy-v2") + if err != nil { + utils.LogError(a.logger, nil, fmt.Sprintf("failed to get inspect keploy container:%v", inspect)) + return err + } + + keployNetworks := inspect.NetworkSettings.Networks + var keployIPv4 string + for n, settings := range keployNetworks { + if n == opts.DockerNetwork { + keployIPv4 = settings.IPAddress //keploy container IP + break + } + } + + ipv4, err := hooks.IPv4ToUint32(keployIPv4) + if err != nil { + return err + } + + var ipv6 [4]uint32 + if opts.IsDocker { + ipv6, err := hooks.ToIPv4MappedIPv6(keployIPv4) + if err != nil { + return fmt.Errorf("failed to convert ipv4:%v to ipv4 mapped ipv6 in docker env:%v", ipv4, err) + } + a.logger.Debug(fmt.Sprintf("IPv4-mapped IPv6 for %s is: %08x:%08x:%08x:%08x\n", keployIPv4, ipv6[0], ipv6[1], ipv6[2], ipv6[3])) + + } + + proxyInfo := structs.ProxyInfo{ + IP4: ipv4, + IP6: ipv6, + Port: 36789, + } + + err = a.Hooks.SendClientProxyInfo(opts.ClientID, proxyInfo) + if err != nil { + return err + } + return nil +} diff --git a/pkg/service/agent/service.go b/pkg/service/agent/service.go new file mode 100644 index 0000000000..97c8295392 --- /dev/null +++ b/pkg/service/agent/service.go @@ -0,0 +1,35 @@ +package agent + +import ( + "context" + + "go.keploy.io/server/v2/pkg/models" +) + +type Service interface { + Setup(ctx context.Context, opts models.SetupOptions) error + GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) + GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) + MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error + SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error + GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) + RegisterClient(ctx context.Context, opts models.SetupOptions) error + DeRegisterClient(ctx context.Context, opts models.UnregisterReq) error +} + +type Options struct { + // Platform Platform + Network string + Container string + SelfTesting bool + Mode models.Mode +} + +// type Platform string + +// var ( +// linux Platform = "linux" +// windows Platform = "windows" +// mac Platform = "mac" +// docker Platform = "docker" +// ) diff --git a/pkg/service/agent/utils.go b/pkg/service/agent/utils.go new file mode 100644 index 0000000000..d62565f354 --- /dev/null +++ b/pkg/service/agent/utils.go @@ -0,0 +1,2 @@ +// Package agent contains utilities for agent. +package agent diff --git a/pkg/service/orchestrator/orchestrator.go b/pkg/service/orchestrator/orchestrator.go index b07150fbde..7a44d018dd 100644 --- a/pkg/service/orchestrator/orchestrator.go +++ b/pkg/service/orchestrator/orchestrator.go @@ -1,5 +1,3 @@ -//go:build linux - // Package orchestrator acts as a main brain for both the record and replay services package orchestrator diff --git a/pkg/service/orchestrator/rerecord.go b/pkg/service/orchestrator/rerecord.go index 2f56eb9ccf..e968ddb10c 100644 --- a/pkg/service/orchestrator/rerecord.go +++ b/pkg/service/orchestrator/rerecord.go @@ -1,5 +1,3 @@ -//go:build linux - package orchestrator import ( @@ -207,7 +205,7 @@ func (o *Orchestrator) replayTests(ctx context.Context, testSet string) (bool, e if utils.IsDockerCmd(cmdType) { host = o.config.ContainerName - userIP, err = o.record.GetContainerIP(ctx, o.config.AppID) + userIP, err = o.replay.GetContainerIP(ctx, o.config.ClientID) if err != nil { utils.LogError(o.logger, err, "failed to get the app ip") return false, err diff --git a/pkg/service/record/record.go b/pkg/service/record/record.go index 78712fa56d..cba30815dc 100755 --- a/pkg/service/record/record.go +++ b/pkg/service/record/record.go @@ -1,5 +1,3 @@ -//go:build linux - // Package record provides functionality for recording and managing test cases and mocks. package record @@ -7,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "time" @@ -49,13 +48,15 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { runAppCtx := context.WithoutCancel(ctx) runAppCtx, runAppCtxCancel := context.WithCancel(runAppCtx) - hookErrGrp, _ := errgroup.WithContext(ctx) - hookCtx := context.WithoutCancel(ctx) - hookCtx, hookCtxCancel := context.WithCancel(hookCtx) - hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) - // reRecordCtx, reRecordCancel := context.WithCancel(ctx) - // defer reRecordCancel() // Cancel the context when the function returns + setupErrGrp, _ := errgroup.WithContext(ctx) + setupCtx := context.WithoutCancel(ctx) + _, setupCtxCancel := context.WithCancel(setupCtx) + setupCtx = context.WithValue(ctx, models.ErrGroupKey, setupErrGrp) + reqErrGrp, _ := errgroup.WithContext(ctx) + reqCtx := context.WithoutCancel(ctx) + _, reqCtxCancel := context.WithCancel(reqCtx) + reqCtx = context.WithValue(ctx, models.ErrGroupKey, reqErrGrp) var stopReason string // defining all the channels and variables required for the record @@ -63,7 +64,7 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { var appErrChan = make(chan models.AppError, 1) var insertTestErrChan = make(chan error, 10) var insertMockErrChan = make(chan error, 10) - var appID uint64 + var clientID uint64 var newTestSetID string var testCount = 0 var mockCountMap = make(map[string]int) @@ -80,26 +81,44 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { } } } + + unregister := models.UnregisterReq{ + ClientID: clientID, + Mode: models.MODE_RECORD, + } + + err := r.instrumentation.UnregisterClient(ctx, unregister) + if err != nil && err != io.EOF { + fmt.Println("error in unregistering client record") + utils.LogError(r.logger, err, "failed to unregister client") + } + runAppCtxCancel() - err := runAppErrGrp.Wait() + err = runAppErrGrp.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop application") } - hookCtxCancel() - err = hookErrGrp.Wait() + + setupCtxCancel() + err = setupErrGrp.Wait() if err != nil { - utils.LogError(r.logger, err, "failed to stop hooks") + utils.LogError(r.logger, err, "failed to stop setup execution, that covers init container") } + err = errGrp.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop recording") } + + reqCtxCancel() + // err = reqErrGrp.Wait() + // if err != nil && err != io.EOF { + // utils.LogError(r.logger, err, "failed to stop request execution") + // } r.telemetry.RecordedTestSuite(newTestSetID, testCount, mockCountMap) }() defer close(appErrChan) - defer close(insertTestErrChan) - defer close(insertMockErrChan) newTestSetID, err := r.GetNextTestSetID(ctx) if err != nil { @@ -115,18 +134,17 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { default: } - // Instrument will setup the environment and start the hooks and proxy - appID, err = r.Instrument(hookCtx) + clientID, err = r.instrumentation.Setup(setupCtx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay, Mode: models.MODE_RECORD, CommandType: r.config.CommandType}) if err != nil { - stopReason = "failed to instrument the application" + stopReason = "failed setting up the environment" utils.LogError(r.logger, err, stopReason) return fmt.Errorf(stopReason) } - r.config.AppID = appID + r.config.ClientID = clientID // fetching test cases and mocks from the application and inserting them into the database - frames, err := r.GetTestAndMockChans(ctx, appID) + frames, err := r.GetTestAndMockChans(reqCtx, clientID) if err != nil { stopReason = "failed to get data frames" utils.LogError(r.logger, err, stopReason) @@ -155,6 +173,9 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { errGrp.Go(func() error { for mock := range frames.Outgoing { + if mock == nil || mock.GetKind() == "" { + continue + } err := r.mockDB.InsertMock(ctx, mock, newTestSetID) if err != nil { if ctx.Err() == context.Canceled { @@ -171,7 +192,7 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { // running the user application runAppErrGrp.Go(func() error { - runAppError = r.instrumentation.Run(runAppCtx, appID, models.RunOptions{}) + runAppError = r.instrumentation.Run(runAppCtx, clientID, models.RunOptions{}) if runAppError.AppErrorType == models.ErrCtxCanceled { return nil } @@ -233,64 +254,91 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { return fmt.Errorf(stopReason) } -func (r *Recorder) Instrument(ctx context.Context) (uint64, error) { - var stopReason string - - // setting up the environment for recording - appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay}) - if err != nil { - stopReason = "failed setting up the environment" - utils.LogError(r.logger, err, stopReason) - return 0, fmt.Errorf(stopReason) - } - r.config.AppID = appID - - // checking for context cancellation as we don't want to start the hooks and proxy if the context is cancelled - select { - case <-ctx.Done(): - return appID, nil - default: - // Starting the hooks and proxy - err = r.instrumentation.Hook(ctx, appID, models.HookOptions{Mode: models.MODE_RECORD, EnableTesting: r.config.EnableTesting, Rules: r.config.BypassRules}) - if err != nil { - stopReason = "failed to start the hooks and proxy" - utils.LogError(r.logger, err, stopReason) - if ctx.Err() == context.Canceled { - return appID, err - } - return appID, fmt.Errorf(stopReason) - } - } - return appID, nil -} - -func (r *Recorder) GetTestAndMockChans(ctx context.Context, appID uint64) (FrameChan, error) { +func (r *Recorder) GetTestAndMockChans(ctx context.Context, clientID uint64) (FrameChan, error) { incomingOpts := models.IncomingOptions{ Filters: r.config.Record.Filters, } - incomingChan, err := r.instrumentation.GetIncoming(ctx, appID, incomingOpts) - if err != nil { - return FrameChan{}, fmt.Errorf("failed to get incoming test cases: %w", err) - } outgoingOpts := models.OutgoingOptions{ Rules: r.config.BypassRules, MongoPassword: r.config.Test.MongoPassword, FallBackOnMiss: r.config.Test.FallBackOnMiss, } - outgoingChan, err := r.instrumentation.GetOutgoing(ctx, appID, outgoingOpts) - if err != nil { - return FrameChan{}, fmt.Errorf("failed to get outgoing mocks: %w", err) + + // Create channels to receive incoming and outgoing data + incomingChan := make(chan *models.TestCase) + outgoingChan := make(chan *models.Mock) + errChan := make(chan error, 2) + + // g, ctx := errgroup.WithContext(ctx) + + g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return FrameChan{}, fmt.Errorf("failed to get error group from context") } + g.Go(func() error { + defer close(incomingChan) + + ch, err := r.instrumentation.GetIncoming(ctx, clientID, incomingOpts) + if err != nil { + errChan <- err + return fmt.Errorf("failed to get incoming test cases: %w", err) + } + for testCase := range ch { + incomingChan <- testCase + } + return nil + }) + + g.Go(func() error { + defer close(outgoingChan) + // create a context without cancel + // change this name to some mockCtx error group + mockErrGrp, _ := errgroup.WithContext(ctx) + mockCtx := context.WithoutCancel(ctx) + mockCtx, mockCtxCancel := context.WithCancel(mockCtx) + + defer func() { + fmt.Println("closing reqCtx") + mockCtxCancel() + err := mockErrGrp.Wait() + if err != nil && err != io.EOF { + utils.LogError(r.logger, err, "failed to stop request execution") + } + }() + + ch, err := r.instrumentation.GetOutgoing(mockCtx, clientID, outgoingOpts) + if err != nil { + r.logger.Error("failed to get outgoing mocks", zap.Error(err)) + errChan <- err + return fmt.Errorf("failed to get outgoing mocks: %w", err) + } + + for mock := range ch { + select { + case <-ctx.Done(): + fmt.Println("context cancelled.....") + if mock != nil { + fmt.Println("mock is not nil") + outgoingChan <- mock + } + return nil + default: + outgoingChan <- mock + } + } + return nil + }) + return FrameChan{ Incoming: incomingChan, Outgoing: outgoingChan, }, nil } -func (r *Recorder) RunApplication(ctx context.Context, appID uint64, opts models.RunOptions) models.AppError { - return r.instrumentation.Run(ctx, appID, opts) +func (r *Recorder) RunApplication(ctx context.Context, clientID uint64, opts models.RunOptions) models.AppError { + return r.instrumentation.Run(ctx, clientID, opts) } func (r *Recorder) GetNextTestSetID(ctx context.Context) (string, error) { @@ -301,6 +349,6 @@ func (r *Recorder) GetNextTestSetID(ctx context.Context) (string, error) { return pkg.NextID(testSetIDs, models.TestSetPattern), nil } -func (r *Recorder) GetContainerIP(ctx context.Context, id uint64) (string, error) { - return r.instrumentation.GetContainerIP(ctx, id) +func (r *Recorder) GetContainerIP(ctx context.Context, clientID uint64) (string, error) { + return r.instrumentation.GetContainerIP(ctx, clientID) } diff --git a/pkg/service/record/service.go b/pkg/service/record/service.go index 700ca53bf2..019913b961 100755 --- a/pkg/service/record/service.go +++ b/pkg/service/record/service.go @@ -10,17 +10,16 @@ type Instrumentation interface { //Setup prepares the environment for the recording Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) //Hook will load hooks and start the proxy server. - Hook(ctx context.Context, id uint64, opts models.HookOptions) error GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) // Run is blocking call and will execute until error Run(ctx context.Context, id uint64, opts models.RunOptions) models.AppError GetContainerIP(ctx context.Context, id uint64) (string, error) + UnregisterClient(ctx context.Context, opts models.UnregisterReq) error } type Service interface { Start(ctx context.Context, reRecord bool) error - GetContainerIP(ctx context.Context, id uint64) (string, error) } type TestDB interface { diff --git a/pkg/service/record/utils.go b/pkg/service/record/utils.go index aa96e8bc5c..6162eca075 100644 --- a/pkg/service/record/utils.go +++ b/pkg/service/record/utils.go @@ -1,3 +1 @@ -//go:build linux - package record diff --git a/pkg/service/replay/replay.go b/pkg/service/replay/replay.go index 37a6308bed..14ad430a4a 100644 --- a/pkg/service/replay/replay.go +++ b/pkg/service/replay/replay.go @@ -81,7 +81,11 @@ func (r *Replayer) Start(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) ctx = context.WithValue(ctx, models.ErrGroupKey, g) - var hookCancel context.CancelFunc + setupErrGrp, _ := errgroup.WithContext(ctx) + _ = context.WithoutCancel(ctx) + setupCtx := context.WithValue(ctx, models.ErrGroupKey, setupErrGrp) + setupCtx, setupCtxCancel := context.WithCancel(setupCtx) + var stopReason = "replay completed successfully" // defering the stop function to stop keploy in case of any error in record or in case of context cancellation @@ -90,12 +94,25 @@ func (r *Replayer) Start(ctx context.Context) error { case <-ctx.Done(): break default: + unregister := models.UnregisterReq{ + ClientID: r.config.ClientID, + Mode: models.MODE_TEST, + } + err := r.instrumentation.UnregisterClient(ctx, unregister) + if err != nil { + fmt.Println("error in unregistering client replay") + utils.LogError(r.logger, err, "failed to unregister client") + } r.logger.Info("stopping Keploy", zap.String("reason", stopReason)) } - if hookCancel != nil { - hookCancel() + + setupCtxCancel() + err := setupErrGrp.Wait() + if err != nil { + utils.LogError(r.logger, err, "failed to stop setup execution, that covers init container") } - err := g.Wait() + + err = g.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop replaying") } @@ -171,8 +188,7 @@ func (r *Replayer) Start(ctx context.Context) error { } } - // Instrument will load the hooks and start the proxy - inst, err := r.Instrument(ctx) + inst, err := r.Instrument(setupCtx) if err != nil { stopReason = fmt.Sprintf("failed to instrument: %v", err) utils.LogError(r.logger, err, stopReason) @@ -182,8 +198,6 @@ func (r *Replayer) Start(ctx context.Context) error { return fmt.Errorf(stopReason) } - hookCancel = inst.HookCancel - var testSetResult bool testRunResult := true abortTestRun := false @@ -220,7 +234,7 @@ func (r *Replayer) Start(ctx context.Context) error { } } - testSetStatus, err := r.RunTestSet(ctx, testSet, testRunID, inst.AppID, false) + testSetStatus, err := r.RunTestSet(ctx, testSet, testRunID, inst.ClientID, false) if err != nil { stopReason = fmt.Sprintf("failed to run test set: %v", err) utils.LogError(r.logger, err, stopReason) @@ -331,33 +345,17 @@ func (r *Replayer) Instrument(ctx context.Context) (*InstrumentState, error) { r.logger.Info("Keploy will not mock the outgoing calls when base path is provided", zap.Any("base path", r.config.Test.BasePath)) return &InstrumentState{}, nil } - appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay}) + // Instrument will setup the environment and start the hooks and proxy + clientID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, CommandType: r.config.CommandType, DockerDelay: r.config.BuildDelay, Mode: models.MODE_TEST}) if err != nil { - if errors.Is(err, context.Canceled) { - return &InstrumentState{}, err - } - return &InstrumentState{}, fmt.Errorf("failed to setup instrumentation: %w", err) + stopReason := "failed setting up the environment" + utils.LogError(r.logger, err, stopReason) + return &InstrumentState{}, fmt.Errorf(stopReason) } - r.config.AppID = appID - var cancel context.CancelFunc - // starting the hooks and proxy - select { - case <-ctx.Done(): - return &InstrumentState{}, context.Canceled - default: - hookCtx := context.WithoutCancel(ctx) - hookCtx, cancel = context.WithCancel(hookCtx) - err = r.instrumentation.Hook(hookCtx, appID, models.HookOptions{Mode: models.MODE_TEST, EnableTesting: r.config.EnableTesting, Rules: r.config.BypassRules}) - if err != nil { - cancel() - if errors.Is(err, context.Canceled) { - return &InstrumentState{}, err - } - return &InstrumentState{}, fmt.Errorf("failed to start the hooks and proxy: %w", err) - } - } - return &InstrumentState{AppID: appID, HookCancel: cancel}, nil + r.config.ClientID = clientID + + return &InstrumentState{ClientID: clientID}, nil } func (r *Replayer) GetNextTestRunID(ctx context.Context) (string, error) { @@ -379,7 +377,7 @@ func (r *Replayer) GetTestCases(ctx context.Context, testID string) ([]*models.T return r.testDB.GetTestCases(ctx, testID) } -func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID string, appID uint64, serveTest bool) (models.TestSetStatus, error) { +func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID string, clientID uint64, serveTest bool) (models.TestSetStatus, error) { // creating error group to manage proper shutdown of all the go routines and to propagate the error to the caller runTestSetErrGrp, runTestSetCtx := errgroup.WithContext(ctx) @@ -475,7 +473,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s cmdType := utils.CmdType(r.config.CommandType) var userIP string - err = r.SetupOrUpdateMocks(runTestSetCtx, appID, testSetID, models.BaseTime, time.Now(), Start) + err = r.SetupOrUpdateMocks(runTestSetCtx, clientID, testSetID, models.BaseTime, time.Now(), Start) if err != nil { return models.TestSetStatusFailed, err } @@ -484,7 +482,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s if !serveTest { runTestSetErrGrp.Go(func() error { defer utils.Recover(r.logger) - appErr = r.RunApplication(runTestSetCtx, appID, models.RunOptions{}) + appErr = r.RunApplication(runTestSetCtx, clientID, models.RunOptions{}) if appErr.AppErrorType == models.ErrCtxCanceled { return nil } @@ -529,7 +527,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } if utils.IsDockerCmd(cmdType) { - userIP, err = r.instrumentation.GetContainerIP(ctx, appID) + userIP, err = r.instrumentation.GetContainerIP(ctx, clientID) if err != nil { return models.TestSetStatusFailed, err } @@ -617,7 +615,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s var loopErr error //No need to handle mocking when basepath is provided - err := r.SetupOrUpdateMocks(runTestSetCtx, appID, testSetID, testCase.HTTPReq.Timestamp, testCase.HTTPResp.Timestamp, Update) + err := r.SetupOrUpdateMocks(runTestSetCtx, clientID, testSetID, testCase.HTTPReq.Timestamp, testCase.HTTPResp.Timestamp, Update) if err != nil { utils.LogError(r.logger, err, "failed to update mocks") break @@ -646,7 +644,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } started := time.Now().UTC() - resp, loopErr := HookImpl.SimulateRequest(runTestSetCtx, appID, testCase, testSetID) + resp, loopErr := HookImpl.SimulateRequest(runTestSetCtx, clientID, testCase, testSetID) if loopErr != nil { utils.LogError(r.logger, err, "failed to simulate request") failure++ @@ -655,7 +653,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s var consumedMocks []string if r.instrument { - consumedMocks, err = r.instrumentation.GetConsumedMocks(runTestSetCtx, appID) + consumedMocks, err = r.instrumentation.GetConsumedMocks(runTestSetCtx, clientID) if err != nil { utils.LogError(r.logger, err, "failed to get consumed filtered mocks") } @@ -876,6 +874,7 @@ func (r *Replayer) SetupOrUpdateMocks(ctx context.Context, appID uint64, testSet } if action == Start { + // api call here - err = r.instrumentation.MockOutgoing(ctx, appID, models.OutgoingOptions{ Rules: r.config.BypassRules, MongoPassword: r.config.Test.MongoPassword, @@ -889,6 +888,7 @@ func (r *Replayer) SetupOrUpdateMocks(ctx context.Context, appID uint64, testSet } } + // this will be sent to the proxy err = r.instrumentation.SetMocks(ctx, appID, filteredMocks, unfilteredMocks) if err != nil { utils.LogError(r.logger, err, "failed to set mocks") @@ -987,8 +987,9 @@ func (r *Replayer) printSummary(_ context.Context, _ bool) { } } -func (r *Replayer) RunApplication(ctx context.Context, appID uint64, opts models.RunOptions) models.AppError { - return r.instrumentation.Run(ctx, appID, opts) +func (r *Replayer) RunApplication(ctx context.Context, clientID uint64, opts models.RunOptions) models.AppError { + fmt.Println("Running the application with clientID: ", clientID) + return r.instrumentation.Run(ctx, clientID, opts) } func (r *Replayer) GetTestSetConf(ctx context.Context, testSet string) (*models.TestSet, error) { @@ -1142,3 +1143,7 @@ func (r *Replayer) DeleteTests(ctx context.Context, testSetID string, testCaseID func SetTestHooks(testHooks TestHooks) { HookImpl = testHooks } + +func (r *Replayer) GetContainerIP(ctx context.Context, id uint64) (string, error) { + return r.instrumentation.GetContainerIP(ctx, id) +} diff --git a/pkg/service/replay/service.go b/pkg/service/replay/service.go index 5ffa0414b1..87cddc2c7e 100644 --- a/pkg/service/replay/service.go +++ b/pkg/service/replay/service.go @@ -12,7 +12,7 @@ type Instrumentation interface { //Setup prepares the environment for the recording Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) //Hook will load hooks and start the proxy server. - Hook(ctx context.Context, id uint64, opts models.HookOptions) error + // Hook(ctx context.Context, id uint64, opts models.HookOptions) error MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error // SetMocks Allows for setting mocks between test runs for better filtering and matching SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error @@ -20,15 +20,16 @@ type Instrumentation interface { GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) // Run is blocking call and will execute until error Run(ctx context.Context, id uint64, opts models.RunOptions) models.AppError - + UnregisterClient(ctx context.Context, opts models.UnregisterReq) error GetContainerIP(ctx context.Context, id uint64) (string, error) } type Service interface { Start(ctx context.Context) error - Instrument(ctx context.Context) (*InstrumentState, error) + // Instrument(ctx context.Context) (*InstrumentState, error) GetNextTestRunID(ctx context.Context) (string, error) GetAllTestSetIDs(ctx context.Context) ([]string, error) + GetContainerIP(ctx context.Context, id uint64) (string, error) RunTestSet(ctx context.Context, testSetID string, testRunID string, appID uint64, serveTest bool) (models.TestSetStatus, error) GetTestSetStatus(ctx context.Context, testRunID string, testSetID string) (models.TestSetStatus, error) GetTestCases(ctx context.Context, testID string) ([]*models.TestCase, error) @@ -89,8 +90,7 @@ type Storage interface { } type InstrumentState struct { - AppID uint64 - HookCancel context.CancelFunc + ClientID uint64 } type MockAction string diff --git a/utils/signal_others.go b/utils/signal_others.go index e450719c76..11ea93a542 100644 --- a/utils/signal_others.go +++ b/utils/signal_others.go @@ -29,15 +29,15 @@ func SendSignal(logger *zap.Logger, pid int, sig syscall.Signal) error { func ExecuteCommand(ctx context.Context, logger *zap.Logger, userCmd string, cancel func(cmd *exec.Cmd) func() error, waitDelay time.Duration) CmdError { // Run the app as the user who invoked sudo - username := os.Getenv("SUDO_USER") + // username := os.Getenv("SUDO_USER") cmd := exec.CommandContext(ctx, "sh", "-c", userCmd) - if username != "" { - // print all environment variables - logger.Debug("env inherited from the cmd", zap.Any("env", os.Environ())) - // Run the command as the user who invoked sudo to preserve the user environment variables and PATH - cmd = exec.CommandContext(ctx, "sudo", "-E", "-u", os.Getenv("SUDO_USER"), "env", "PATH="+os.Getenv("PATH"), "sh", "-c", userCmd) - } + // if username != "" { + // // print all environment variables + // logger.Debug("env inherited from the cmd", zap.Any("env", os.Environ())) + // // Run the command as the user who invoked sudo to preserve the user environment variables and PATH + // cmd = exec.CommandContext(ctx, "sudo", "-E", "-u", os.Getenv("SUDO_USER"), "env", "PATH="+os.Getenv("PATH"), "sh", "-c", userCmd) + // } // Set the cancel function for the command cmd.Cancel = cancel(cmd) @@ -53,7 +53,7 @@ func ExecuteCommand(ctx context.Context, logger *zap.Logger, userCmd string, can cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - logger.Debug("", zap.Any("executing cli", cmd.String())) + logger.Info("", zap.Any("executing cli", cmd.String())) err := cmd.Start() if err != nil { diff --git a/utils/utils.go b/utils/utils.go index 04a29ec606..39c1790a44 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "debug/elf" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -26,6 +27,7 @@ import ( "golang.org/x/text/language" "github.com/getsentry/sentry-go" + "github.com/google/uuid" netLib "github.com/shirou/gopsutil/v3/net" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -193,6 +195,16 @@ func DeleteFileIfNotExists(logger *zap.Logger, name string) (err error) { return nil } +func GenerateID() uint64 { + // Random AppId uint64 will be generated and maintain in a map and return the id to client + newUUID := uuid.New() + + // app id will be sent by the client. + // Convert the first 8 bytes of the UUID to an int64 + id := int64(binary.BigEndian.Uint64(newUUID[:8])) + return uint64(id) +} + type GitHubRelease struct { TagName string `json:"tag_name"` Body string `json:"body"` @@ -910,3 +922,18 @@ func IsFileEmpty(filePath string) (bool, error) { } return fileInfo.Size() == 0, nil } + +func GetCurrentBinaryPath() (string, error) { + executable, err := os.Executable() + if err != nil { + return "", err + } + + // Resolve the full path (to avoid issues with symbolic links) + executablePath, err := filepath.EvalSymlinks(executable) + if err != nil { + return "", err + } + + return executablePath, nil +}