diff --git a/.github/ISSUE_TEMPLATE/--bug-report.yaml b/.github/ISSUE_TEMPLATE/--bug-report.yaml index 5c446ea4bd..be3df6812b 100755 --- a/.github/ISSUE_TEMPLATE/--bug-report.yaml +++ b/.github/ISSUE_TEMPLATE/--bug-report.yaml @@ -39,7 +39,7 @@ body: id: environment attributes: label: "🧱 Your Environment" - description: "What exact version of Appwrite? What Appwrite SDK and version are you using? Is your environment customized in any way?" + description: "What exact version of Keploy? Is your environment customized in any way?" placeholder: "I use AWS for ..." validations: required: false diff --git a/.github/workflows/test_workflow_scripts/golang-docker.sh b/.github/workflows/test_workflow_scripts/golang-docker.sh index 618425eb0e..2eb6c89c40 100755 --- a/.github/workflows/test_workflow_scripts/golang-docker.sh +++ b/.github/workflows/test_workflow_scripts/golang-docker.sh @@ -22,10 +22,29 @@ docker build -t gin-mongo . docker rm -f ginApp 2>/dev/null || true container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -57,7 +76,11 @@ send_request(){ # Wait for 5 seconds for keploy to record the tcs and mocks. sleep 5 + # sudo docker rm -f keploy-v2 + # sleep 5 + # sudo docker rm -f keploy-init container_kill + # sleep 5 wait } @@ -69,32 +92,41 @@ for i in {1..2}; do if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 +# container_kill +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + +echo "Starting the test phase..." # Start the keploy in test mode. test_container="ginApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c 'docker run -p8080:8080 --net keploy-network --name ginApp_test gin-mongo' --containerName "$test_container" --apiTimeout 60 --delay 20 --generate-github-actions=false &> "${test_container}.txt" +# container_kill +# sudo docker rm -f keploy-v2 + if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/golang-linux.sh b/.github/workflows/test_workflow_scripts/golang-linux.sh index 42fccf5943..96bfe3dd65 100644 --- a/.github/workflows/test_workflow_scripts/golang-linux.sh +++ b/.github/workflows/test_workflow_scripts/golang-linux.sh @@ -26,9 +26,14 @@ sed -i 's/ports: 0/ports: 27017/' "$config_file" # Remove any preexisting keploy tests and mocks. rm -rf keploy/ +echo "Starting the pipeline..." + # Build the binary. go build -o ginApp +# Start keploy agent in the background + +echo "Keploy agent started" send_request(){ sleep 10 @@ -70,47 +75,54 @@ send_request(){ sudo kill $pid } - for i in {1..2}; do + echo "Starting iteration ${i}" app_name="javaApp_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & - sudo -E env PATH="$PATH" ./../../keployv2 record -c "./ginApp" &> "${app_name}.txt" + sudo -E env PATH="$PATH" ./../../keployv2 record -c "./ginApp" &> "${app_name}.txt" --debug if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done +sleep 10 +echo "Starting the pipeline for test mode..." + +sudo ./../../keployv2 agent & + +echo "Keploy agent started for test mode" + +sleep 8 + # Start the gin-mongo app in test mode. -sudo -E env PATH="$PATH" ./../../keployv2 test -c "./ginApp" --delay 7 &> test_logs.txt +sudo -E env PATH="$PATH" ./../../keployv2 test -c "./ginApp" --delay 7 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true - # Get the test results from the testReport file. -for i in {0..1} -do +for i in {0..1}; do # Define the report file for each test set report_file="./keploy/reports/test-run-0/test-set-$i-report.yaml" @@ -135,4 +147,13 @@ if [ "$all_passed" = true ]; then else cat "test_logs.txt" exit 1 -fi \ No newline at end of file +fi + +# Finally, stop the keploy agent +agent_pid=$(pgrep -f 'keployv2 agent') +if [ -z "$agent_pid" ]; then + echo "Keploy agent process not found." +else + echo "Stopping keploy agent with PID: $agent_pid" + sudo kill $agent_pid +fi diff --git a/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh b/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh index 2033d1d687..323615d26f 100644 --- a/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh +++ b/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh @@ -47,17 +47,19 @@ send_request() { for i in {1..2}; do app_name="urlShort_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH="$PATH" ./../../keployv2 record -c "./urlShort" --generateGithubActions=false &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait @@ -65,18 +67,20 @@ for i in {1..2}; do done # Start the gin-mongo app in test mode. +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH="$PATH" ./../../keployv2 test -c "./urlShort" --delay 7 --generateGithubActions=false &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/java-linux.sh b/.github/workflows/test_workflow_scripts/java-linux.sh index 6150cbf45f..5222e8a9b8 100644 --- a/.github/workflows/test_workflow_scripts/java-linux.sh +++ b/.github/workflows/test_workflow_scripts/java-linux.sh @@ -65,34 +65,40 @@ for i in {1..2}; do # Start keploy in record mode. mvn clean install -Dmaven.test.skip=true app_name="javaApp_${i}" + sudo ./../../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH=$PATH ./../../../keployv2 record -c 'java -jar target/spring-petclinic-rest-3.0.2.jar' &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done +sleep 5 +echo "Starting test sessions" +sudo ./../../../keployv2 agent & +sleep 5 # Start keploy in test mode. sudo -E env PATH=$PATH ./../../../keployv2 test -c 'java -jar target/spring-petclinic-rest-3.0.2.jar' --delay 20 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/node-docker.sh b/.github/workflows/test_workflow_scripts/node-docker.sh index 6d21e70a7d..d32e06fc56 100755 --- a/.github/workflows/test_workflow_scripts/node-docker.sh +++ b/.github/workflows/test_workflow_scripts/node-docker.sh @@ -1,5 +1,6 @@ #!/bin/bash + source ./../../.github/workflows/test_workflow_scripts/test-iid.sh # Start the docker container. @@ -13,10 +14,29 @@ sudo rm -rf keploy/ docker build -t node-app:1.0 . container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -51,7 +71,7 @@ send_request(){ curl -X GET http://localhost:8000/students # Wait for 5 seconds for keploy to record the tcs and mocks. - sleep 5 + sleep 10 container_kill wait } @@ -64,31 +84,39 @@ for i in {1..2}; do if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 + +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + +echo "Starting the test phase..." + # Start keploy in test mode. test_container="nodeApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c "docker run -p8000:8000 --rm --name $test_container --network keploy-network node-app:1.0" --containerName "$test_container" --apiTimeout 30 --delay 30 --generate-github-actions=false &> "${test_container}.txt" + if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi # Monitor Docker logs for race conditions during testing. if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/node-linux.sh b/.github/workflows/test_workflow_scripts/node-linux.sh index 5c4e3f2184..874171857a 100755 --- a/.github/workflows/test_workflow_scripts/node-linux.sh +++ b/.github/workflows/test_workflow_scripts/node-linux.sh @@ -47,17 +47,19 @@ send_request(){ # Record and test sessions in a loop for i in {1..2}; do app_name="nodeApp_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH=$PATH ./../../keployv2 record -c 'npm start' &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait @@ -67,44 +69,52 @@ done mocks_file="keploy/test-set-0/tests/test-5.yaml" sed -i 's/"page":1/"page":4/' "$mocks_file" +sleep 5 +echo "Starting test sessions" +sudo ./../../keployv2 agent & +sleep 5 # Test modes and result checking sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 &> test_logs1.txt if grep "ERROR" "test_logs1.txt"; then echo "Error found in pipeline..." cat "test_logs1.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs1.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs1.txt" - exit 1 + # exit 1 fi +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 --testsets test-set-0 &> test_logs2.txt if grep "ERROR" "test_logs2.txt"; then echo "Error found in pipeline..." cat "test_logs2.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs2.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs2.txt" - exit 1 + # exit 1 fi sed -i 's/selectedTests: {}/selectedTests: {"test-set-0": ["test-1", "test-2"]}/' "./keploy.yml" +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --apiTimeout 30 --delay 10 &> test_logs3.txt if grep "ERROR" "test_logs3.txt"; then echo "Error found in pipeline..." cat "test_logs3.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs3.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs3.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/python-docker.sh b/.github/workflows/test_workflow_scripts/python-docker.sh index 1beeb8b6a5..529c393b14 100755 --- a/.github/workflows/test_workflow_scripts/python-docker.sh +++ b/.github/workflows/test_workflow_scripts/python-docker.sh @@ -1,5 +1,6 @@ #!/bin/bash + source ./../../.github/workflows/test_workflow_scripts/test-iid.sh # Start mongo before starting keploy. @@ -16,10 +17,29 @@ sleep 5 # Allow time for configuration to apply container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -55,30 +75,35 @@ for i in {1..2}; do if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done + +sleep 4 +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + # Testing phase test_container="flashApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c "docker run -p8080:8080 --net keploy-network --name $test_container flask-app:1.0" --containerName "$test_container" --apiTimeout 60 --delay 20 --generate-github-actions=false &> "${test_container}.txt" if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/python-linux.sh b/.github/workflows/test_workflow_scripts/python-linux.sh index acd249b0a2..76b9b6d182 100644 --- a/.github/workflows/test_workflow_scripts/python-linux.sh +++ b/.github/workflows/test_workflow_scripts/python-linux.sh @@ -61,35 +61,41 @@ send_request(){ # Record and Test cycles for i in {1..2}; do app_name="flaskApp_${i}" + sudo ./../../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH="$PATH" ./../../../keployv2 record -c "python3 manage.py runserver" &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done + +sudo ./../../../keployv2 agent & +sleep 5 + # Testing phase sudo -E env PATH="$PATH" ./../../../keployv2 test -c "python3 manage.py runserver" --delay 10 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/Dockerfile b/Dockerfile index 08d8ced5b6..3e281db68a 100755 --- a/Dockerfile +++ b/Dockerfile @@ -29,13 +29,13 @@ RUN apt-get install -y ca-certificates curl sudo && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Install Docker engine -RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ - sh get-docker.sh && \ - rm get-docker.sh +# # Install Docker engine +# RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ +# sh get-docker.sh && \ +# rm get-docker.sh # Install docker-compose to PATH -RUN apt install docker-compose -y +# RUN apt install docker-compose -y # Copy the keploy binary and the entrypoint script from the build container COPY --from=build /app/keploy /app/keploy @@ -48,4 +48,4 @@ RUN sed -i 's/\r$//' /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Set the entrypoint -ENTRYPOINT ["/app/entrypoint.sh", "/app/keploy"] \ No newline at end of file +ENTRYPOINT ["/app/entrypoint.sh", "/app/keploy", "agent","--is-docker", "--port", "8096","--proxy-port", "36789"] \ No newline at end of file diff --git a/READMEes-Es.md b/READMEes-Es.md index 623b128e56..1d99f10444 100644 --- a/READMEes-Es.md +++ b/READMEes-Es.md @@ -153,7 +153,7 @@ Keploy se puede utilizar en 1 && unixPath[1] == ':' { - unixPath = unixPath[2:] - } - return unixPath -} diff --git a/cli/provider/service.go b/cli/provider/service.go index 50338ff368..75dd53a4a9 100644 --- a/cli/provider/service.go +++ b/cli/provider/service.go @@ -47,6 +47,8 @@ func (n *ServiceProvider) GetService(ctx context.Context, cmd string) (interface return utgen.NewUnitTestGenerator(n.cfg, tel, n.auth, n.logger) case "record", "test", "mock", "normalize", "templatize", "rerecord", "contract": return Get(ctx, cmd, n.cfg, n.logger, tel, n.auth) + case "agent": + return GetAgent(ctx, cmd, n.cfg, n.logger, n.auth) default: return nil, errors.New("invalid command") } diff --git a/cli/record.go b/cli/record.go index dca17fd389..b9197ecbbc 100755 --- a/cli/record.go +++ b/cli/record.go @@ -28,6 +28,7 @@ func Record(ctx context.Context, logger *zap.Logger, _ *config.Config, serviceFa utils.LogError(logger, err, "failed to get service") return nil } + var record recordSvc.Service var ok bool if record, ok = svc.(recordSvc.Service); !ok { diff --git a/config/config.go b/config/config.go index 59711a4e8f..3080ae5342 100644 --- a/config/config.go +++ b/config/config.go @@ -10,11 +10,12 @@ import ( type Config struct { Path string `json:"path" yaml:"path" mapstructure:"path"` - AppID uint64 `json:"appId" yaml:"appId" mapstructure:"appId"` + ClientID uint64 `json:"clientID" yaml:"clientID" mapstructure:"clientID"` AppName string `json:"appName" yaml:"appName" mapstructure:"appName"` Command string `json:"command" yaml:"command" mapstructure:"command"` Templatize Templatize `json:"templatize" yaml:"templatize" mapstructure:"templatize"` Port uint32 `json:"port" yaml:"port" mapstructure:"port"` + ServerPort uint32 `json:"serverPort" yaml:"serverPort" mapstructure:"serverPort"` DNSPort uint32 `json:"dnsPort" yaml:"dnsPort" mapstructure:"dnsPort"` ProxyPort uint32 `json:"proxyPort" yaml:"proxyPort" mapstructure:"proxyPort"` Debug bool `json:"debug" yaml:"debug" mapstructure:"debug"` @@ -37,12 +38,12 @@ type Config struct { KeployNetwork string `json:"keployNetwork" yaml:"keployNetwork" mapstructure:"keployNetwork"` CommandType string `json:"cmdType" yaml:"cmdType" mapstructure:"cmdType"` Contract Contract `json:"contract" yaml:"contract" mapstructure:"contract"` - - InCi bool `json:"inCi" yaml:"inCi" mapstructure:"inCi"` - InstallationID string `json:"-" yaml:"-" mapstructure:"-"` - Version string `json:"-" yaml:"-" mapstructure:"-"` - APIServerURL string `json:"-" yaml:"-" mapstructure:"-"` - GitHubClientID string `json:"-" yaml:"-" mapstructure:"-"` + Agent Agent `json:"agent" yaml:"agent" mapstructure:"agent"` + InCi bool `json:"inCi" yaml:"inCi" mapstructure:"inCi"` + InstallationID string `json:"-" yaml:"-" mapstructure:"-"` + Version string `json:"-" yaml:"-" mapstructure:"-"` + APIServerURL string `json:"-" yaml:"-" mapstructure:"-"` + GitHubClientID string `json:"-" yaml:"-" mapstructure:"-"` } type UtGen struct { @@ -130,6 +131,12 @@ type Test struct { UpdateTemplate bool `json:"updateTemplate" yaml:"updateTemplate" mapstructure:"updateTemplate"` } +type Agent struct { + IsDocker bool `json:"isDocker" yaml:"isDocker" mapstructure:"isDocker"` + Port uint32 `json:"port" yaml:"port" mapstructure:"port"` + ProxyPort uint32 `json:"proxyPort" yaml:"proxyPort" mapstructure:"proxyPort"` +} + type Language string // String is used both by fmt.Print and by Cobra in help text diff --git a/go.mod b/go.mod index db9f9f9738..fe8d8c45ee 100755 --- a/go.mod +++ b/go.mod @@ -2,14 +2,13 @@ module go.keploy.io/server/v2 go 1.22.0 -replace github.com/jackc/pgproto3/v2 => github.com/keploy/pgproto3/v2 v2.0.5 +replace github.com/jackc/pgproto3/v2 => github.com/keploy/pgproto3/v2 v2.0.7 require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cilium/ebpf v0.13.2 github.com/cloudflare/cfssl v1.6.4 - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.4+incompatible + github.com/docker/docker v27.2.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.16.0 @@ -23,9 +22,9 @@ require ( github.com/spf13/cobra v1.8.0 go.mongodb.org/mongo-driver v1.11.6 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sys v0.21.0 - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/sys v0.25.0 + google.golang.org/protobuf v1.34.2 // indirect ) require ( @@ -78,7 +77,7 @@ require ( ) require ( - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -98,8 +97,8 @@ require ( github.com/zmap/zlint/v3 v3.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.26.0 - golang.org/x/text v0.16.0 + golang.org/x/net v0.29.0 + golang.org/x/text v0.18.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect k8s.io/klog/v2 v2.120.1 // indirect ) @@ -112,6 +111,8 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 github.com/emirpasic/gods v1.18.1 github.com/getsentry/sentry-go v0.28.1 + github.com/go-chi/chi/v5 v5.1.0 + github.com/go-chi/render v1.0.3 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 github.com/jackc/pgproto3/v2 v2.3.2 @@ -122,13 +123,26 @@ require ( github.com/xdg-go/scram v1.1.1 github.com/xdg-go/stringprep v1.0.4 github.com/yudai/gojsondiff v1.0.0 - golang.org/x/sync v0.7.0 - golang.org/x/term v0.21.0 + golang.org/x/sync v0.8.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/kustomize/kyaml v0.17.2 ) -require github.com/perimeterx/marshmallow v1.1.5 // indirect +require ( + github.com/ajg/form v1.5.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect +) require ( github.com/alecthomas/chroma v0.10.0 // indirect diff --git a/go.sum b/go.sum index 56768491b0..a8a3e58555 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= @@ -16,6 +18,8 @@ github.com/aymanbagabas/go-osc52 v1.0.3 h1:DTwqENW7X9arYimJrPeGZcV0ln14sGMt3pHZs github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cilium/ebpf v0.13.2 h1:uhLimLX+jF9BTPPvoCUYh/mBeoONkjgaJ9w9fn0mRj4= @@ -34,12 +38,12 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.4+incompatible h1:s/LVDftw9hjblvqIeTiGYXBCD95nOEEl7qRsRrIOuQI= -github.com/docker/docker v24.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= +github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -48,6 +52,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -58,10 +64,17 @@ github.com/getkin/kin-openapi v0.126.0 h1:c2cSgLnAsS0xYfKsgt5oBV6MYRM/giU8/RtwUY github.com/getkin/kin-openapi v0.126.0/go.mod h1:7mONz8IwmSRg6RttPu6v8U/OJ+gr+J99qSFNjPGSQqw= github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= +github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -113,6 +126,9 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -130,8 +146,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/k0kubun/pp/v3 v3.2.0 h1:h33hNTZ9nVFNP3u2Fsgz8JXiF5JINoZfFq4SvKJwNcs= github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapdz1EwA= -github.com/keploy/pgproto3/v2 v2.0.5 h1:8spdNKZ+nOnHVxiimDsqulBRN6viPXPghkA7xppnzJ8= -github.com/keploy/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/keploy/pgproto3/v2 v2.0.7 h1:cBQo5N3zZsQTnQC/c6gLqjrPSSIeao7bInNVLdniyr8= +github.com/keploy/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -172,6 +188,8 @@ github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby v26.0.2+incompatible h1:t41TD3nRvK8E6bZFJdKrmNlH8Xe3epTmdNXf/mnfLKk= github.com/moby/moby v26.0.2+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -330,6 +348,22 @@ github.com/zmap/zlint/v3 v3.1.0 h1:WjVytZo79m/L1+/Mlphl09WBob6YTGljN5IGWZFpAv0= github.com/zmap/zlint/v3 v3.1.0/go.mod h1:L7t8s3sEKkb0A2BxGy1IWrxt1ZATa1R4QfJZaQOD3zU= go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -344,8 +378,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -365,16 +399,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -401,21 +435,21 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -430,14 +464,21 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= +google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index fb28072f00..4aff58f1de 100755 --- a/main.go +++ b/main.go @@ -14,8 +14,9 @@ import ( userDb "go.keploy.io/server/v2/pkg/platform/yaml/configdb/user" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/utils/log" + //pprof for debugging - //_ "net/http/pprof" + _ "net/http/pprof" ) // version is the version of the server and will be injected during build by ldflags, same with dsn @@ -30,7 +31,7 @@ func main() { // Uncomment the following code to enable pprof for debugging // go func() { // fmt.Println("Starting pprof server for debugging...") - // err := http.ListenAndServe("localhost:6060", nil) + // err := http.ListenAndServe("0.0.0.0:6060", nil) // if err != nil { // fmt.Println("Failed to start the pprof server for debugging", err) // return diff --git a/pkg/core/core_others.go b/pkg/agent/core_others.go similarity index 98% rename from pkg/core/core_others.go rename to pkg/agent/core_others.go index 8446cde7a6..703626a3c8 100644 --- a/pkg/core/core_others.go +++ b/pkg/agent/core_others.go @@ -1,7 +1,7 @@ //go:build !linux // Package core provides functionality for managing core functionalities in Keploy. -package core +package agent import ( "context" diff --git a/pkg/core/hooks/README.md b/pkg/agent/hooks/README.md similarity index 100% rename from pkg/core/hooks/README.md rename to pkg/agent/hooks/README.md diff --git a/pkg/core/hooks/bpf_arm64_bpfel.go b/pkg/agent/hooks/bpf_arm64_bpfel.go similarity index 97% rename from pkg/core/hooks/bpf_arm64_bpfel.go rename to pkg/agent/hooks/bpf_arm64_bpfel.go index df9cc8dbec..b43c75e212 100644 --- a/pkg/core/hooks/bpf_arm64_bpfel.go +++ b/pkg/agent/hooks/bpf_arm64_bpfel.go @@ -95,17 +95,18 @@ type bpfMapSpecs struct { ConnInfoMap *ebpf.MapSpec `ebpf:"conn_info_map"` CurrentSockMap *ebpf.MapSpec `ebpf:"current_sock_map"` DestInfoMap *ebpf.MapSpec `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.MapSpec `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.MapSpec `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.MapSpec `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.MapSpec `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.MapSpec `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.MapSpec `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.MapSpec `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.MapSpec `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.MapSpec `ebpf:"socket_data_event_buffer_heap"` SocketDataEvents *ebpf.MapSpec `ebpf:"socket_data_events"` SocketOpenEvents *ebpf.MapSpec `ebpf:"socket_open_events"` TaskStructMap *ebpf.MapSpec `ebpf:"task_struct_map"` + TestbenchInfoMap *ebpf.MapSpec `ebpf:"testbench_info_map"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -135,17 +136,18 @@ type bpfMaps struct { ConnInfoMap *ebpf.Map `ebpf:"conn_info_map"` CurrentSockMap *ebpf.Map `ebpf:"current_sock_map"` DestInfoMap *ebpf.Map `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.Map `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.Map `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.Map `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.Map `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.Map `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.Map `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.Map `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.Map `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.Map `ebpf:"socket_data_event_buffer_heap"` SocketDataEvents *ebpf.Map `ebpf:"socket_data_events"` SocketOpenEvents *ebpf.Map `ebpf:"socket_open_events"` TaskStructMap *ebpf.Map `ebpf:"task_struct_map"` + TestbenchInfoMap *ebpf.Map `ebpf:"testbench_info_map"` } func (m *bpfMaps) Close() error { @@ -158,17 +160,18 @@ func (m *bpfMaps) Close() error { m.ConnInfoMap, m.CurrentSockMap, m.DestInfoMap, - m.DockerAppRegistrationMap, m.KeployAgentKernelPidMap, m.KeployAgentRegistrationMap, m.KeployClientKernelPidMap, m.KeployClientRegistrationMap, + m.KeployProxyInfo, m.RedirectProxyMap, m.SocketCloseEvents, m.SocketDataEventBufferHeap, m.SocketDataEvents, m.SocketOpenEvents, m.TaskStructMap, + m.TestbenchInfoMap, ) } diff --git a/pkg/agent/hooks/bpf_arm64_bpfel.o b/pkg/agent/hooks/bpf_arm64_bpfel.o new file mode 100644 index 0000000000..bb7ceb1ea9 Binary files /dev/null and b/pkg/agent/hooks/bpf_arm64_bpfel.o differ diff --git a/pkg/core/hooks/bpf_x86_bpfel.go b/pkg/agent/hooks/bpf_x86_bpfel.go similarity index 97% rename from pkg/core/hooks/bpf_x86_bpfel.go rename to pkg/agent/hooks/bpf_x86_bpfel.go index cf8f135116..40f590d5ad 100644 --- a/pkg/core/hooks/bpf_x86_bpfel.go +++ b/pkg/agent/hooks/bpf_x86_bpfel.go @@ -95,17 +95,18 @@ type bpfMapSpecs struct { ConnInfoMap *ebpf.MapSpec `ebpf:"conn_info_map"` CurrentSockMap *ebpf.MapSpec `ebpf:"current_sock_map"` DestInfoMap *ebpf.MapSpec `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.MapSpec `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.MapSpec `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.MapSpec `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.MapSpec `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.MapSpec `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.MapSpec `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.MapSpec `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.MapSpec `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.MapSpec `ebpf:"socket_data_event_buffer_heap"` SocketDataEvents *ebpf.MapSpec `ebpf:"socket_data_events"` SocketOpenEvents *ebpf.MapSpec `ebpf:"socket_open_events"` TaskStructMap *ebpf.MapSpec `ebpf:"task_struct_map"` + TestbenchInfoMap *ebpf.MapSpec `ebpf:"testbench_info_map"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -135,17 +136,18 @@ type bpfMaps struct { ConnInfoMap *ebpf.Map `ebpf:"conn_info_map"` CurrentSockMap *ebpf.Map `ebpf:"current_sock_map"` DestInfoMap *ebpf.Map `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.Map `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.Map `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.Map `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.Map `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.Map `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.Map `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.Map `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.Map `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.Map `ebpf:"socket_data_event_buffer_heap"` SocketDataEvents *ebpf.Map `ebpf:"socket_data_events"` SocketOpenEvents *ebpf.Map `ebpf:"socket_open_events"` TaskStructMap *ebpf.Map `ebpf:"task_struct_map"` + TestbenchInfoMap *ebpf.Map `ebpf:"testbench_info_map"` } func (m *bpfMaps) Close() error { @@ -158,17 +160,18 @@ func (m *bpfMaps) Close() error { m.ConnInfoMap, m.CurrentSockMap, m.DestInfoMap, - m.DockerAppRegistrationMap, m.KeployAgentKernelPidMap, m.KeployAgentRegistrationMap, m.KeployClientKernelPidMap, m.KeployClientRegistrationMap, + m.KeployProxyInfo, m.RedirectProxyMap, m.SocketCloseEvents, m.SocketDataEventBufferHeap, m.SocketDataEvents, m.SocketOpenEvents, m.TaskStructMap, + m.TestbenchInfoMap, ) } diff --git a/pkg/agent/hooks/bpf_x86_bpfel.o b/pkg/agent/hooks/bpf_x86_bpfel.o new file mode 100644 index 0000000000..9f1ca8db68 Binary files /dev/null and b/pkg/agent/hooks/bpf_x86_bpfel.o differ diff --git a/pkg/core/hooks/conn/README.md b/pkg/agent/hooks/conn/README.md similarity index 100% rename from pkg/core/hooks/conn/README.md rename to pkg/agent/hooks/conn/README.md diff --git a/pkg/core/hooks/conn/conn.go b/pkg/agent/hooks/conn/conn.go similarity index 100% rename from pkg/core/hooks/conn/conn.go rename to pkg/agent/hooks/conn/conn.go diff --git a/pkg/core/hooks/conn/factory.go b/pkg/agent/hooks/conn/factory.go similarity index 87% rename from pkg/core/hooks/conn/factory.go rename to pkg/agent/hooks/conn/factory.go index 1156922814..0f9d4ec479 100755 --- a/pkg/core/hooks/conn/factory.go +++ b/pkg/agent/hooks/conn/factory.go @@ -43,7 +43,7 @@ func NewFactory(inactivityThreshold time.Duration, logger *zap.Logger) *Factory // ProcessActiveTrackers iterates over all conn the trackers and checks if they are complete. If so, it captures the ingress call and // deletes the tracker. If the tracker is inactive for a long time, it deletes it. -func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *models.TestCase, opts models.IncomingOptions) { +func (factory *Factory) ProcessActiveTrackers(ctx context.Context, testMap *sync.Map, opts models.IncomingOptions) { factory.mutex.Lock() defer factory.mutex.Unlock() var trackersToDelete []ID @@ -52,9 +52,10 @@ func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *model case <-ctx.Done(): return default: - ok, requestBuf, responseBuf, reqTimestampTest, resTimestampTest := tracker.IsComplete() + ok, requestBuf, responseBuf, reqTimestampTest, resTimestampTest, clientID := tracker.IsComplete() if ok { - + fmt.Println("Processing the tracker with key: ", connID) + fmt.Println("Request Buffer::::::::: ", string(requestBuf)) if len(requestBuf) == 0 || len(responseBuf) == 0 { factory.logger.Warn("failed processing a request due to invalid request or response", zap.Any("Request Size", len(requestBuf)), zap.Any("Response Size", len(responseBuf))) continue @@ -70,7 +71,23 @@ func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *model utils.LogError(factory.logger, err, "failed to parse the http response from byte array", zap.Any("responseBuf", responseBuf)) continue } - capture(ctx, factory.logger, t, parsedHTTPReq, parsedHTTPRes, reqTimestampTest, resTimestampTest, opts) + + //get the channel from the test map + // failed to get the channel from the test map, if the client id is not found + t, ok := testMap.Load(clientID) + if !ok { + factory.logger.Error("failed to get the channel from the test map") + continue + } + + // type assert the channel + tc, ok := t.(chan *models.TestCase) + if !ok { + factory.logger.Error("failed to type assert the channel from the test map") + continue + } + + capture(ctx, factory.logger, tc, parsedHTTPReq, parsedHTTPRes, reqTimestampTest, resTimestampTest, opts) } else if tracker.IsInactive(factory.inactivityThreshold) { trackersToDelete = append(trackersToDelete, connID) @@ -80,6 +97,7 @@ func (factory *Factory) ProcessActiveTrackers(ctx context.Context, t chan *model // Delete all the processed trackers. for _, key := range trackersToDelete { + fmt.Println("Deleting the tracker with key: ", key) delete(factory.connections, key) } } @@ -104,6 +122,7 @@ func capture(_ context.Context, logger *zap.Logger, t chan *models.TestCase, req return } + fmt.Println("Request Body::::::: ", string(reqBody)) defer func() { err := resp.Body.Close() if err != nil { diff --git a/pkg/core/hooks/conn/socket.go b/pkg/agent/hooks/conn/socket.go similarity index 79% rename from pkg/core/hooks/conn/socket.go rename to pkg/agent/hooks/conn/socket.go index d7af88596f..cffb0ca4cf 100644 --- a/pkg/core/hooks/conn/socket.go +++ b/pkg/agent/hooks/conn/socket.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "sync" "time" "unsafe" @@ -26,18 +27,19 @@ import ( var eventAttributesSize = int(unsafe.Sizeof(SocketDataEvent{})) // ListenSocket starts the socket event listeners -func ListenSocket(ctx context.Context, l *zap.Logger, openMap, dataMap, closeMap *ebpf.Map, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - t := make(chan *models.TestCase, 500) +func ListenSocket(ctx context.Context, l *zap.Logger, clientID uint64, testMap *sync.Map, openMap, dataMap, closeMap *ebpf.Map, opts models.IncomingOptions) error { + err := initRealTimeOffset() if err != nil { utils.LogError(l, err, "failed to initialize real time offset") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } c := NewFactory(time.Minute, l) g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) if !ok { - return nil, errors.New("failed to get the error group from the context") + return errors.New("failed to get the error group from the context") } + fmt.Println("Starting the socket listener", c.connections) g.Go(func() error { defer utils.Recover(l) go func() { @@ -45,35 +47,49 @@ func ListenSocket(ctx context.Context, l *zap.Logger, openMap, dataMap, closeMap for { select { case <-ctx.Done(): + fmt.Println("Context Done in ListenSocket") return default: // TODO refactor this to directly consume the events from the maps - c.ProcessActiveTrackers(ctx, t, opts) + c.ProcessActiveTrackers(ctx, testMap, opts) time.Sleep(100 * time.Millisecond) } } }() <-ctx.Done() - close(t) + + //get the channel from test map and close it + t, ok := testMap.Load(clientID) + if ok { + tc, ok := t.(chan *models.TestCase) + if ok { + // Close the channel when the context is done + close(tc) + } else { + println("Failed to type assert the channel from the test map") + } + } + return nil }) err = open(ctx, c, l, openMap) if err != nil { utils.LogError(l, err, "failed to start open socket listener") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } - err = data(ctx, c, l, dataMap) + + err = data(ctx, c, l, dataMap, clientID) if err != nil { utils.LogError(l, err, "failed to start data socket listener") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } err = exit(ctx, c, l, closeMap) if err != nil { utils.LogError(l, err, "failed to start close socket listener") - return nil, errors.New("failed to start socket listeners") + return errors.New("failed to start socket listeners") } - return t, err + return err } func open(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { @@ -128,7 +144,7 @@ func open(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { return nil } -func data(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { +func data(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map, clientID uint64) error { r, err := ringbuf.NewReader(m) if err != nil { utils.LogError(l, nil, "failed to create ring buffer of socketDataEvent") @@ -176,6 +192,21 @@ func data(ctx context.Context, c *Factory, l *zap.Logger, m *ebpf.Map) error { l.Debug(fmt.Sprintf("Request EntryTimestamp :%v\n", convertUnixNanoToTime(event.EntryTimestampNano))) } + // if event.ClientID == clientID { + // fmt.Println("Event received for the keploy test client") + // continue + // } + + // if event.ClientID != id { + // // log the expected client id and the received client id + // l.Info(fmt.Sprintf("Expected ClientID: %v, Received ClientID: %v", id, event.ClientID)) + + // continue + // } + + fmt.Println("SocketDataEvent-1: ", event.ClientID) + fmt.Printf("Direction: %v\n", event.Direction, "Actual Message: ", string(event.Msg[:event.MsgSize])) + c.GetOrCreate(event.ConnID).AddDataEvent(event) } }() diff --git a/pkg/core/hooks/conn/tracker.go b/pkg/agent/hooks/conn/tracker.go similarity index 96% rename from pkg/core/hooks/conn/tracker.go rename to pkg/agent/hooks/conn/tracker.go index 14bd7e9475..6c2de8d9dd 100755 --- a/pkg/core/hooks/conn/tracker.go +++ b/pkg/agent/hooks/conn/tracker.go @@ -66,6 +66,9 @@ type Tracker struct { reqTimestamps []time.Time isNewRequest bool + + //Client Id's array + clientIDs []uint64 } func NewTracker(connID ID, logger *zap.Logger) *Tracker { @@ -107,7 +110,7 @@ func (conn *Tracker) decRecordTestCount() { } // IsComplete checks if the current conn has valid request & response info to capture and also returns the request and response data buffer. -func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { +func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time, uint64) { conn.mutex.Lock() defer conn.mutex.Unlock() @@ -166,7 +169,8 @@ func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { if len(conn.userReqs) > 0 && len(conn.userResps) > 0 { //validated request, response requestBuf = conn.userReqs[0] responseBuf = conn.userResps[0] - + fmt.Println("Request Buffer::::::::: ", string(requestBuf)) + fmt.Println("Response Buffer::::::::: ", string(responseBuf)) //popping out the current request & response data conn.userReqs = conn.userReqs[1:] conn.userResps = conn.userResps[1:] @@ -233,6 +237,7 @@ func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { conn.logger.Debug("unverified recording", zap.Any("recordTraffic", recordTraffic)) } + var clientID uint64 // Checking if record traffic is recorded and request & response timestamp is captured or not. if recordTraffic { if len(conn.reqTimestamps) > 0 { @@ -252,9 +257,16 @@ func (conn *Tracker) IsComplete() (bool, []byte, []byte, time.Time, time.Time) { } conn.logger.Debug(fmt.Sprintf("TestRequestTimestamp:%v || TestResponseTimestamp:%v", reqTimestamps, respTimestamp)) + + //popping out the client id + if len(conn.clientIDs) > 0 { + clientID = conn.clientIDs[0] + conn.clientIDs = conn.clientIDs[1:] + } + } - return recordTraffic, requestBuf, responseBuf, reqTimestamps, respTimestamp + return recordTraffic, requestBuf, responseBuf, reqTimestamps, respTimestamp, clientID } // reset resets the conn's request and response data buffers. @@ -301,6 +313,9 @@ func (conn *Tracker) AddDataEvent(event SocketDataEvent) { // This is to ensure that we capture the response timestamp for the first chunk of the response. if !conn.isNewRequest { conn.isNewRequest = true + + // set the client id + conn.clientIDs = append(conn.clientIDs, event.ClientID) } // Assign the size of the message to the variable msgLengt diff --git a/pkg/core/hooks/conn/util.go b/pkg/agent/hooks/conn/util.go similarity index 98% rename from pkg/core/hooks/conn/util.go rename to pkg/agent/hooks/conn/util.go index a1b80f25f8..e13ff9a968 100755 --- a/pkg/core/hooks/conn/util.go +++ b/pkg/agent/hooks/conn/util.go @@ -10,7 +10,7 @@ import ( "time" "go.keploy.io/server/v2/config" - proxyHttp "go.keploy.io/server/v2/pkg/core/proxy/integrations/http" + proxyHttp "go.keploy.io/server/v2/pkg/agent/proxy/integrations/http" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/hooks/hooks.go b/pkg/agent/hooks/hooks.go similarity index 78% rename from pkg/core/hooks/hooks.go rename to pkg/agent/hooks/hooks.go index 2f589abf69..51e434609f 100644 --- a/pkg/core/hooks/hooks.go +++ b/pkg/agent/hooks/hooks.go @@ -20,41 +20,49 @@ import ( "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/rlimit" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/hooks/conn" - "go.keploy.io/server/v2/pkg/core/hooks/structs" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks/conn" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" "go.keploy.io/server/v2/pkg/models" "go.uber.org/zap" ) func NewHooks(logger *zap.Logger, cfg *config.Config) *Hooks { + return &Hooks{ logger: logger, - sess: core.NewSessions(), + sess: agent.NewSessions(), m: sync.Mutex{}, proxyIP4: "127.0.0.1", proxyIP6: [4]uint32{0000, 0000, 0000, 0001}, - proxyPort: cfg.ProxyPort, + proxyPort: cfg.Agent.ProxyPort, dnsPort: cfg.DNSPort, + TestMap: &sync.Map{}, + isLoaded: false, } } type Hooks struct { logger *zap.Logger - sess *core.Sessions + sess *agent.Sessions proxyIP4 string proxyIP6 [4]uint32 proxyPort uint32 dnsPort uint32 - - m sync.Mutex + TestMap *sync.Map + m sync.Mutex + isLoaded bool // eBPF C shared maps - clientRegistrationMap *ebpf.Map - agentRegistartionMap *ebpf.Map - dockerAppRegistrationMap *ebpf.Map - redirectProxyMap *ebpf.Map + clientRegistrationMap *ebpf.Map + agentRegistartionMap *ebpf.Map + + redirectProxyMap *ebpf.Map + proxyInfoMap *ebpf.Map //-------------- + // test bench maps + tbenchFilterPid *ebpf.Map + tbenchFilterPort *ebpf.Map // eBPF C shared objectsobjects // ebpf objects and events socket link.Link @@ -87,16 +95,15 @@ type Hooks struct { objects bpfObjects writev link.Link writevRet link.Link - appID uint64 } -func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { +func (h *Hooks) Load(ctx context.Context, id uint64, opts agent.HookCfg) error { - h.sess.Set(id, &core.Session{ + h.sess.Set(id, &agent.Session{ ID: id, }) - err := h.load(ctx, opts) + err := h.load(opts) if err != nil { return err } @@ -119,7 +126,7 @@ func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { return nil } -func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { +func (h *Hooks) load(opts agent.HookCfg) error { // Allow the current process to lock memory for eBPF resources. if err := rlimit.RemoveMemlock(); err != nil { utils.LogError(h.logger, err, "failed to lock memory for eBPF resources") @@ -142,7 +149,9 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.redirectProxyMap = objs.RedirectProxyMap h.clientRegistrationMap = objs.KeployClientRegistrationMap h.agentRegistartionMap = objs.KeployAgentRegistrationMap - h.dockerAppRegistrationMap = objs.DockerAppRegistrationMap + h.proxyInfoMap = objs.KeployProxyInfo + h.tbenchFilterPid = objs.TestbenchInfoMap + h.objects = objs // --------------- @@ -408,28 +417,17 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.logger.Info("keploy initialized and probes added to the kernel.") - var clientInfo structs.ClientInfo = structs.ClientInfo{} - - switch opts.Mode { - case models.MODE_RECORD: - clientInfo.Mode = uint32(1) - case models.MODE_TEST: - clientInfo.Mode = uint32(2) - default: - clientInfo.Mode = uint32(0) - } - //sending keploy pid to kernel to get filtered - inode, err := getSelfInodeNumber() - if err != nil { - utils.LogError(h.logger, err, "failed to get inode of the keploy process") - return err - } + // inode, err := getSelfInodeNumber() + // if err != nil { + // utils.LogError(h.logger, err, "failed to get inode of the keploy process") + // return err + // } - clientInfo.KeployClientInode = inode - clientInfo.KeployClientNsPid = uint32(os.Getpid()) - clientInfo.IsKeployClientRegistered = uint32(0) - h.logger.Debug("Keploy Pid sent successfully...") + // clientInfo.KeployClientInode = inode + // clientInfo.KeployClientNsPid = uint32(os.Getpid()) + // clientInfo.IsKeployClientRegistered = uint32(0) + // h.logger.Info("Keploy Pid sent successfully...") if opts.IsDocker { h.proxyIP4 = opts.KeployIPV4 @@ -443,55 +441,133 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.logger.Debug("proxy ips", zap.String("ipv4", h.proxyIP4), zap.Any("ipv6", h.proxyIP6)) - proxyIP, err := IPv4ToUint32(h.proxyIP4) + var agentInfo structs.AgentInfo = structs.AgentInfo{} + agentInfo.KeployAgentNsPid = uint32(os.Getpid()) + agentInfo.KeployAgentInode, err = GetSelfInodeNumber() if err != nil { - return fmt.Errorf("failed to convert ip string:[%v] to 32-bit integer", opts.KeployIPV4) + utils.LogError(h.logger, err, "failed to get inode of the keploy process") + return err } - var agentInfo structs.AgentInfo = structs.AgentInfo{} + agentInfo.DNSPort = int32(h.dnsPort) - agentInfo.ProxyInfo = structs.ProxyInfo{ - IP4: proxyIP, - IP6: h.proxyIP6, - Port: h.proxyPort, + // if opts.IsDocker { + // clientInfo.IsDockerApp = uint32(1) + // } else { + // clientInfo.IsDockerApp = uint32(0) + // } + + // ports := GetPortToSendToKernel(ctx, opts.Rules) + // for i := 0; i < 10; i++ { + // if len(ports) <= i { + // clientInfo.PassThroughPorts[i] = -1 + // continue + // } + // clientInfo.PassThroughPorts[i] = int32(ports[i]) + // } + + // for sending client pid to kernel + // fmt.Println("Sending client info to kernel...", clientInfo) + // err = h.SendClientInfo(opts.AppID, clientInfo) + // if err != nil { + // h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + // return err + // } + err = h.SendAgentInfo(agentInfo) + if err != nil { + h.logger.Error("failed to send agent info to the ebpf program", zap.Error(err)) + return err } - agentInfo.DNSPort = int32(h.dnsPort) + return nil +} - if opts.IsDocker { - clientInfo.IsDockerApp = uint32(1) +func (h *Hooks) Record(ctx context.Context, clientID uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + tc := make(chan *models.TestCase, 1) + kctx := context.WithoutCancel(ctx) + h.TestMap.Store(clientID, tc) + if !h.isLoaded { + err := conn.ListenSocket(kctx, h.logger, clientID, h.TestMap, h.objects.SocketOpenEvents, h.objects.SocketDataEvents, h.objects.SocketCloseEvents, opts) + if err != nil { + return nil, err + } + h.isLoaded = true } else { - clientInfo.IsDockerApp = uint32(0) - } - - ports := GetPortToSendToKernel(ctx, opts.Rules) - for i := 0; i < 10; i++ { - if len(ports) <= i { - clientInfo.PassThroughPorts[i] = -1 - continue + t, ok := h.TestMap.Load(clientID) + if ok { + tc, ok = t.(chan *models.TestCase) + if ok { + // Close the channel when the context is done + } else { + h.logger.Error("Failed to type assert the channel from the test map") + } } - clientInfo.PassThroughPorts[i] = int32(ports[i]) } - err = h.SendClientInfo(opts.AppID, clientInfo) + return tc, nil +} + +func (h *Hooks) SendKeployClientInfo(clientID uint64, clientInfo structs.ClientInfo) error { + + err := h.SendClientInfo(clientID, clientInfo) if err != nil { h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) return err } - err = h.SendAgentInfo(agentInfo) + + return nil +} + +// SendKeployPids is used to send keploy recordServer(key-0) or testServer(key-1) Pid to the ebpf program +func (h *Hooks) SendKeployPids(key models.ModeKey, tb structs.TestBenchInfo) error { + fmt.Println("Test bench info in SendKeployPids", tb) + err := h.tbenchFilterPid.Update(key, &tb, ebpf.UpdateAny) if err != nil { - h.logger.Error("failed to send agent info to the ebpf program", zap.Error(err)) return err } + return nil +} + +// For keploy test bench +// The below function is used to send the keploy record binary server port to the ebpf so that the flow first reaches to the keploy record proxy and then keploy test proxy + +// SendKeployPorts is used to send keploy recordServer(key-0) or testServer(key-1) Port to the ebpf program +func (h *Hooks) SendKeployPorts(key models.ModeKey, port uint32) error { + + err := h.tbenchFilterPort.Update(key, &port, ebpf.UpdateAny) + if err != nil { + return err + } + return nil +} + +func (h *Hooks) DeleteKeployClientInfo(id uint64) error { + err := h.DeleteClientInfo(id) + if err != nil { + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + return err + } + return nil +} +func (h *Hooks) SendClientProxyInfo(clientID uint64, proxyInfo structs.ProxyInfo) error { + err := h.SendProxyInfo(clientID, proxyInfo) + if err != nil { + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + return err + } return nil } -func (h *Hooks) Record(ctx context.Context, _ uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - // TODO use the session to get the app id - // and then use the app id to get the test cases chan - // and pass that to eBPF consumers/listeners - return conn.ListenSocket(ctx, h.logger, h.objects.SocketOpenEvents, h.objects.SocketDataEvents, h.objects.SocketCloseEvents, opts) +func (h *Hooks) SendKtInfo(ctx context.Context, tb structs.TestBenchInfo) error { + + fmt.Println("Test bench info", tb) + err := h.SendKeployPids(models.TestKey, tb) + if err != nil { + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + return err + } + return nil } func (h *Hooks) unLoad(_ context.Context) { diff --git a/pkg/agent/hooks/kernelComm.go b/pkg/agent/hooks/kernelComm.go new file mode 100644 index 0000000000..84cdfe9e95 --- /dev/null +++ b/pkg/agent/hooks/kernelComm.go @@ -0,0 +1,124 @@ +//go:build linux + +package hooks + +import ( + "context" + + "github.com/cilium/ebpf" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" +) + +// Get Used by proxy +func (h *Hooks) Get(_ context.Context, srcPort uint16) (*agent.NetworkAddress, error) { + d, err := h.GetDestinationInfo(srcPort) + if err != nil { + return nil, err + } + // TODO : need to implement eBPF code to differentiate between different apps + // s, ok := h.sess.Get(d.ClientID) + // if !ok { + // return nil, fmt.Errorf("session not found") + // } + + return &agent.NetworkAddress{ + ClientID: d.ClientID, + Version: d.IPVersion, + IPv4Addr: d.DestIP4, + IPv6Addr: d.DestIP6, + Port: d.DestPort, + }, nil +} + +// GetDestinationInfo retrieves destination information associated with a source port. +func (h *Hooks) GetDestinationInfo(srcPort uint16) (*structs.DestInfo, error) { + h.m.Lock() + defer h.m.Unlock() + destInfo := structs.DestInfo{ + ClientID: 0, + } + if err := h.redirectProxyMap.Lookup(srcPort, &destInfo); err != nil { + return nil, err + } + return &destInfo, nil +} + +func (h *Hooks) Delete(_ context.Context, srcPort uint16) error { + return h.CleanProxyEntry(srcPort) +} + +func (h *Hooks) CleanProxyEntry(srcPort uint16) error { + h.m.Lock() + defer h.m.Unlock() + err := h.redirectProxyMap.Delete(srcPort) + if err != nil { + utils.LogError(h.logger, err, "failed to remove entry from redirect proxy map") + return err + } + h.logger.Debug("successfully removed entry from redirect proxy map", zap.Any("(Key)/SourcePort", srcPort)) + return nil +} + +func (h *Hooks) SendClientInfo(id uint64, clientInfo structs.ClientInfo) error { + h.m.Lock() + defer h.m.Unlock() + err := h.clientRegistrationMap.Update(id, clientInfo, ebpf.UpdateAny) + if err != nil { + utils.LogError(h.logger, err, "failed to send the app info to the ebpf program") + return err + } + return nil +} + +func (h *Hooks) DeleteClientInfo(id uint64) error { + h.m.Lock() + defer h.m.Unlock() + err := h.clientRegistrationMap.Delete(id) + if err != nil { + utils.LogError(h.logger, err, "failed to send the app info to the ebpf program") + return err + } + h.logger.Info("successfully removed the client info from the ebpf program with clientId", zap.Any("clientId", id)) + return nil +} + +// SendProxyInfo sends the network information to the kernel +func (h *Hooks) SendProxyInfo(id uint64, proxInfo structs.ProxyInfo) error { + h.m.Lock() + defer h.m.Unlock() + err := h.proxyInfoMap.Update(id, proxInfo, ebpf.UpdateAny) + if err != nil { + utils.LogError(h.logger, err, "failed to send the proxy info to the ebpf program") + return err + } + return nil +} + +// func (h *Hooks) SendTestBenchInfo(key models.ModeKey, tb structs.TestBenchInfo) error { +// fmt.Println("Test bench info", tb) +// if key == models.RecordKey { +// h.logger.Info("Sending test bench info for record mode") +// } else { +// h.logger.Info("Sending test bench info for test mode") +// } + +// err := h.tbenchFilterPid.Update(uint32(0), tb, ebpf.UpdateAny) +// if err != nil { +// utils.LogError(h.logger, err, "failed to send the test bench info to the ebpf program") +// return err +// } +// return nil +// } + +func (h *Hooks) SendAgentInfo(agentInfo structs.AgentInfo) error { + key := 0 + err := h.agentRegistartionMap.Update(uint32(key), agentInfo, ebpf.UpdateAny) + if err != nil { + utils.LogError(h.logger, err, "failed to send the agent info to the ebpf program") + return err + } + return nil +} diff --git a/pkg/core/hooks/structs/structs.go b/pkg/agent/hooks/structs/structs.go similarity index 50% rename from pkg/core/hooks/structs/structs.go rename to pkg/agent/hooks/structs/structs.go index 3e9483d203..6ca06265f5 100755 --- a/pkg/core/hooks/structs/structs.go +++ b/pkg/agent/hooks/structs/structs.go @@ -5,15 +5,6 @@ package structs type BpfSpinLock struct{ Val uint32 } -// struct dest_info_t -// { -// u32 ip_version; -// u32 dest_ip4; -// u32 dest_ip6[4]; -// u32 dest_port; -// u32 kernelPid; -// }; - type DestInfo struct { IPVersion uint32 DestIP4 uint32 @@ -23,13 +14,6 @@ type DestInfo struct { ClientID uint64 } -// struct proxy_info -// { -// u32 ip4; -// u32 ip6[4]; -// u32 port; -// }; - type ProxyInfo struct { IP4 uint32 IP6 [4]uint32 @@ -41,17 +25,6 @@ type DockerAppInfo struct { ClientID uint64 } -// struct app_info -// { -// u32 keploy_client_ns_pid; -// u64 keploy_client_inode; -// u64 app_inode; -// u32 mode; -// u32 is_docker_app; -// u32 is_keploy_client_registered; // whether the client is registered or not -// s32 pass_through_ports[PASS_THROUGH_ARRAY_SIZE]; -// }; - type ClientInfo struct { KeployClientInode uint64 KeployClientNsPid uint32 @@ -59,19 +32,17 @@ type ClientInfo struct { IsDockerApp uint32 IsKeployClientRegistered uint32 PassThroughPorts [10]int32 + AppInode uint64 } -// struct agent_info -// { -// u32 keploy_agent_ns_pid; -// u32 keploy_agent_inode; -// struct proxy_info proxy_info; -// s32 dns_port; -// }; - type AgentInfo struct { KeployAgentNsPid uint32 DNSPort int32 KeployAgentInode uint64 - ProxyInfo ProxyInfo +} + +type TestBenchInfo struct { + KTestClientPID uint32 + KTestAgentPID uint32 + KRecordAgentPID uint32 } diff --git a/pkg/core/hooks/util.go b/pkg/agent/hooks/util.go similarity index 96% rename from pkg/core/hooks/util.go rename to pkg/agent/hooks/util.go index f28a9ff485..8a33f8e002 100644 --- a/pkg/core/hooks/util.go +++ b/pkg/agent/hooks/util.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package hooks @@ -7,6 +7,7 @@ import ( "context" "encoding/binary" "errors" + "fmt" "net" "os" "path/filepath" @@ -28,6 +29,7 @@ func IPv4ToUint32(ipStr string) (uint32, error) { } return 0, errors.New("not a valid IPv4 address") } + fmt.Println("failed to parse IP address", ipStr) return 0, errors.New("failed to parse IP address") } @@ -97,7 +99,7 @@ func detectCgroupPath(logger *zap.Logger) (string, error) { return "", errors.New("cgroup2 not mounted") } -func getSelfInodeNumber() (uint64, error) { +func GetSelfInodeNumber() (uint64, error) { p := filepath.Join("/proc", "self", "ns", "pid") f, err := os.Stat(p) diff --git a/pkg/core/proxy/README.md b/pkg/agent/proxy/README.md similarity index 100% rename from pkg/core/proxy/README.md rename to pkg/agent/proxy/README.md diff --git a/pkg/core/proxy/dns.go b/pkg/agent/proxy/dns.go similarity index 100% rename from pkg/core/proxy/dns.go rename to pkg/agent/proxy/dns.go diff --git a/pkg/core/proxy/integrations/README.md b/pkg/agent/proxy/integrations/README.md similarity index 100% rename from pkg/core/proxy/integrations/README.md rename to pkg/agent/proxy/integrations/README.md diff --git a/pkg/core/proxy/integrations/generic/README.md b/pkg/agent/proxy/integrations/generic/README.md similarity index 100% rename from pkg/core/proxy/integrations/generic/README.md rename to pkg/agent/proxy/integrations/generic/README.md diff --git a/pkg/core/proxy/integrations/generic/decode.go b/pkg/agent/proxy/integrations/generic/decode.go similarity index 95% rename from pkg/core/proxy/integrations/generic/decode.go rename to pkg/agent/proxy/integrations/generic/decode.go index d7b0af8c84..13708e6263 100644 --- a/pkg/core/proxy/integrations/generic/decode.go +++ b/pkg/agent/proxy/integrations/generic/decode.go @@ -9,9 +9,9 @@ import ( "net" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/generic/encode.go b/pkg/agent/proxy/integrations/generic/encode.go similarity index 95% rename from pkg/core/proxy/integrations/generic/encode.go rename to pkg/agent/proxy/integrations/generic/encode.go index 14e85c49d3..c25c106afa 100644 --- a/pkg/core/proxy/integrations/generic/encode.go +++ b/pkg/agent/proxy/integrations/generic/encode.go @@ -11,14 +11,14 @@ import ( "strconv" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) -func encodeGeneric(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func encodeGeneric(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions, clientClose chan bool) error { var genericRequests []models.Payload @@ -73,6 +73,9 @@ func encodeGeneric(ctx context.Context, logger *zap.Logger, reqBuf []byte, clien logger.Debug("the iteration for the generic request starts", zap.Any("genericReqs", len(genericRequests)), zap.Any("genericResps", len(genericResponses))) for { select { + case <-clientClose: + mocks <- &models.Mock{} + return ctx.Err() case <-ctx.Done(): if !prevChunkWasReq && len(genericRequests) > 0 && len(genericResponses) > 0 { genericRequestsCopy := make([]models.Payload, len(genericRequests)) diff --git a/pkg/core/proxy/integrations/generic/generic.go b/pkg/agent/proxy/integrations/generic/generic.go similarity index 86% rename from pkg/core/proxy/integrations/generic/generic.go rename to pkg/agent/proxy/integrations/generic/generic.go index 02f4759af9..cf0bb9480f 100755 --- a/pkg/core/proxy/integrations/generic/generic.go +++ b/pkg/agent/proxy/integrations/generic/generic.go @@ -6,8 +6,8 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -32,7 +32,7 @@ func (g *Generic) MatchType(_ context.Context, _ []byte) bool { return false } -func (g *Generic) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (g *Generic) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := g.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) @@ -41,7 +41,7 @@ func (g *Generic) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn return err } - err = encodeGeneric(ctx, logger, reqBuf, src, dst, mocks, opts) + err = encodeGeneric(ctx, logger, reqBuf, src, dst, mocks, opts, clientClose) if err != nil { utils.LogError(logger, err, "failed to encode the generic message into the yaml") return err diff --git a/pkg/core/proxy/integrations/generic/match.go b/pkg/agent/proxy/integrations/generic/match.go similarity index 97% rename from pkg/core/proxy/integrations/generic/match.go rename to pkg/agent/proxy/integrations/generic/match.go index 6ce7fc7005..6f182044e3 100755 --- a/pkg/core/proxy/integrations/generic/match.go +++ b/pkg/agent/proxy/integrations/generic/match.go @@ -8,9 +8,9 @@ import ( "fmt" "math" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" ) diff --git a/pkg/core/proxy/integrations/grpc/decode.go b/pkg/agent/proxy/integrations/grpc/decode.go similarity index 92% rename from pkg/core/proxy/integrations/grpc/decode.go rename to pkg/agent/proxy/integrations/grpc/decode.go index ffa42761a1..d6635e5f0a 100644 --- a/pkg/core/proxy/integrations/grpc/decode.go +++ b/pkg/agent/proxy/integrations/grpc/decode.go @@ -7,7 +7,7 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/grpc/encode.go b/pkg/agent/proxy/integrations/grpc/encode.go similarity index 93% rename from pkg/core/proxy/integrations/grpc/encode.go rename to pkg/agent/proxy/integrations/grpc/encode.go index 39528758ab..571de348b9 100644 --- a/pkg/core/proxy/integrations/grpc/encode.go +++ b/pkg/agent/proxy/integrations/grpc/encode.go @@ -7,14 +7,14 @@ import ( "io" "net" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) -func encodeGrpc(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func encodeGrpc(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions, clientClose chan bool) error { // Send the client preface to the server. This should be the first thing sent from the client. _, err := destConn.Write(reqBuf) @@ -72,6 +72,9 @@ func encodeGrpc(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientCo }) select { + case <-clientClose: + mocks <- &models.Mock{} + return ctx.Err() case <-ctx.Done(): return ctx.Err() case err := <-errCh: diff --git a/pkg/core/proxy/integrations/grpc/frame.go b/pkg/agent/proxy/integrations/grpc/frame.go similarity index 100% rename from pkg/core/proxy/integrations/grpc/frame.go rename to pkg/agent/proxy/integrations/grpc/frame.go diff --git a/pkg/core/proxy/integrations/grpc/grpc.go b/pkg/agent/proxy/integrations/grpc/grpc.go similarity index 87% rename from pkg/core/proxy/integrations/grpc/grpc.go rename to pkg/agent/proxy/integrations/grpc/grpc.go index 40ae191510..11ca390730 100644 --- a/pkg/core/proxy/integrations/grpc/grpc.go +++ b/pkg/agent/proxy/integrations/grpc/grpc.go @@ -7,8 +7,8 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -35,7 +35,7 @@ func (g *Grpc) MatchType(_ context.Context, reqBuf []byte) bool { return bytes.HasPrefix(reqBuf[:], []byte("PRI * HTTP/2")) } -func (g *Grpc) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (g *Grpc) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := g.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) @@ -44,7 +44,7 @@ func (g *Grpc) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, m return err } - err = encodeGrpc(ctx, logger, reqBuf, src, dst, mocks, opts) + err = encodeGrpc(ctx, logger, reqBuf, src, dst, mocks, opts, clientClose) if err != nil { utils.LogError(logger, err, "failed to encode the grpc message into the yaml") return err diff --git a/pkg/core/proxy/integrations/grpc/match.go b/pkg/agent/proxy/integrations/grpc/match.go similarity index 97% rename from pkg/core/proxy/integrations/grpc/match.go rename to pkg/agent/proxy/integrations/grpc/match.go index c76b68878f..33a5309bfd 100644 --- a/pkg/core/proxy/integrations/grpc/match.go +++ b/pkg/agent/proxy/integrations/grpc/match.go @@ -6,7 +6,7 @@ import ( "context" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.uber.org/zap" "go.keploy.io/server/v2/pkg/models" diff --git a/pkg/core/proxy/integrations/grpc/stream.go b/pkg/agent/proxy/integrations/grpc/stream.go similarity index 100% rename from pkg/core/proxy/integrations/grpc/stream.go rename to pkg/agent/proxy/integrations/grpc/stream.go diff --git a/pkg/core/proxy/integrations/grpc/transcoder.go b/pkg/agent/proxy/integrations/grpc/transcoder.go similarity index 99% rename from pkg/core/proxy/integrations/grpc/transcoder.go rename to pkg/agent/proxy/integrations/grpc/transcoder.go index d59d149748..918b206b8e 100644 --- a/pkg/core/proxy/integrations/grpc/transcoder.go +++ b/pkg/agent/proxy/integrations/grpc/transcoder.go @@ -7,7 +7,7 @@ import ( "context" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/grpc/util.go b/pkg/agent/proxy/integrations/grpc/util.go similarity index 100% rename from pkg/core/proxy/integrations/grpc/util.go rename to pkg/agent/proxy/integrations/grpc/util.go diff --git a/pkg/core/proxy/integrations/http/README.md b/pkg/agent/proxy/integrations/http/README.md similarity index 100% rename from pkg/core/proxy/integrations/http/README.md rename to pkg/agent/proxy/integrations/http/README.md diff --git a/pkg/core/proxy/integrations/http/decode.go b/pkg/agent/proxy/integrations/http/decode.go similarity index 98% rename from pkg/core/proxy/integrations/http/decode.go rename to pkg/agent/proxy/integrations/http/decode.go index dabef6c319..93833ebeb6 100644 --- a/pkg/core/proxy/integrations/http/decode.go +++ b/pkg/agent/proxy/integrations/http/decode.go @@ -15,8 +15,8 @@ import ( "strconv" "go.keploy.io/server/v2/pkg" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/http/encode.go b/pkg/agent/proxy/integrations/http/encode.go similarity index 98% rename from pkg/core/proxy/integrations/http/encode.go rename to pkg/agent/proxy/integrations/http/encode.go index f551c11d04..3ff260e8f5 100644 --- a/pkg/core/proxy/integrations/http/encode.go +++ b/pkg/agent/proxy/integrations/http/encode.go @@ -13,8 +13,8 @@ import ( "golang.org/x/sync/errgroup" - "go.keploy.io/server/v2/pkg/core/proxy/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/http/http.go b/pkg/agent/proxy/integrations/http/http.go similarity index 94% rename from pkg/core/proxy/integrations/http/http.go rename to pkg/agent/proxy/integrations/http/http.go index 4521864536..c627f4e6fb 100755 --- a/pkg/core/proxy/integrations/http/http.go +++ b/pkg/agent/proxy/integrations/http/http.go @@ -14,8 +14,8 @@ import ( "strconv" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/pkg" @@ -60,10 +60,10 @@ func (h *HTTP) MatchType(_ context.Context, buf []byte) bool { return isHTTP } -func (h *HTTP) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (h *HTTP) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := h.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) - h.logger.Debug("Recording the outgoing http call in record mode") + h.logger.Info("Recording the outgoing http call in record mode") reqBuf, err := util.ReadInitialBuf(ctx, logger, src) if err != nil { @@ -80,7 +80,7 @@ func (h *HTTP) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, m func (h *HTTP) MockOutgoing(ctx context.Context, src net.Conn, dstCfg *models.ConditionalDstCfg, mockDb integrations.MockMemDb, opts models.OutgoingOptions) error { logger := h.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) - h.logger.Debug("Mocking the outgoing http call in test mode") + h.logger.Info("Mocking the outgoing http call in test mode") reqBuf, err := util.ReadInitialBuf(ctx, logger, src) if err != nil { diff --git a/pkg/core/proxy/integrations/http/match.go b/pkg/agent/proxy/integrations/http/match.go similarity index 98% rename from pkg/core/proxy/integrations/http/match.go rename to pkg/agent/proxy/integrations/http/match.go index 48d62e7e2e..0fc5d9af36 100644 --- a/pkg/core/proxy/integrations/http/match.go +++ b/pkg/agent/proxy/integrations/http/match.go @@ -14,8 +14,8 @@ import ( "strings" "github.com/agnivade/levenshtein" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/http/util.go b/pkg/agent/proxy/integrations/http/util.go similarity index 99% rename from pkg/core/proxy/integrations/http/util.go rename to pkg/agent/proxy/integrations/http/util.go index 0de57abf4e..9b2ee03c18 100644 --- a/pkg/core/proxy/integrations/http/util.go +++ b/pkg/agent/proxy/integrations/http/util.go @@ -16,7 +16,7 @@ import ( "strings" "time" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/integrations.go b/pkg/agent/proxy/integrations/integrations.go similarity index 94% rename from pkg/core/proxy/integrations/integrations.go rename to pkg/agent/proxy/integrations/integrations.go index eddffb33f4..0e26c3d856 100644 --- a/pkg/core/proxy/integrations/integrations.go +++ b/pkg/agent/proxy/integrations/integrations.go @@ -31,7 +31,7 @@ var Registered = make(map[string]Initializer) type Integrations interface { MatchType(ctx context.Context, reqBuf []byte) bool - RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error + RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error MockOutgoing(ctx context.Context, src net.Conn, dstCfg *models.ConditionalDstCfg, mockDb MockMemDb, opts models.OutgoingOptions) error } diff --git a/pkg/core/proxy/integrations/mongo/README.md b/pkg/agent/proxy/integrations/mongo/README.md similarity index 100% rename from pkg/core/proxy/integrations/mongo/README.md rename to pkg/agent/proxy/integrations/mongo/README.md diff --git a/pkg/core/proxy/integrations/mongo/command.go b/pkg/agent/proxy/integrations/mongo/command.go similarity index 100% rename from pkg/core/proxy/integrations/mongo/command.go rename to pkg/agent/proxy/integrations/mongo/command.go diff --git a/pkg/core/proxy/integrations/mongo/decode.go b/pkg/agent/proxy/integrations/mongo/decode.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/decode.go rename to pkg/agent/proxy/integrations/mongo/decode.go index 4961b7b1df..2819802fe3 100644 --- a/pkg/core/proxy/integrations/mongo/decode.go +++ b/pkg/agent/proxy/integrations/mongo/decode.go @@ -12,8 +12,8 @@ import ( "strconv" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.mongodb.org/mongo-driver/bson" diff --git a/pkg/core/proxy/integrations/mongo/encode.go b/pkg/agent/proxy/integrations/mongo/encode.go similarity index 97% rename from pkg/core/proxy/integrations/mongo/encode.go rename to pkg/agent/proxy/integrations/mongo/encode.go index 963a0fcecb..bb1c5b79e5 100644 --- a/pkg/core/proxy/integrations/mongo/encode.go +++ b/pkg/agent/proxy/integrations/mongo/encode.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sync/errgroup" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -21,7 +21,7 @@ import ( // encodeMongo records the outgoing mongo messages of the client connection, // decodes the wiremessage binary and writes readable string // to the yaml file. -func (m *Mongo) encodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func (m *Mongo) encodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, clientClose chan bool, _ models.OutgoingOptions) error { errCh := make(chan error, 1) @@ -271,6 +271,10 @@ func (m *Mongo) encodeMongo(ctx context.Context, logger *zap.Logger, reqBuf []by }) select { + case <-clientClose: + fmt.Println("client connection is closed from the mongo parser") + mocks <- &models.Mock{} + return ctx.Err() case <-ctx.Done(): return ctx.Err() case err := <-errCh: diff --git a/pkg/core/proxy/integrations/mongo/match.go b/pkg/agent/proxy/integrations/mongo/match.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/match.go rename to pkg/agent/proxy/integrations/mongo/match.go index 6f13ea0f52..177b3c3246 100644 --- a/pkg/core/proxy/integrations/mongo/match.go +++ b/pkg/agent/proxy/integrations/mongo/match.go @@ -8,7 +8,7 @@ import ( "reflect" "strings" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/utils" "go.mongodb.org/mongo-driver/bson" diff --git a/pkg/core/proxy/integrations/mongo/mongo.go b/pkg/agent/proxy/integrations/mongo/mongo.go similarity index 94% rename from pkg/core/proxy/integrations/mongo/mongo.go rename to pkg/agent/proxy/integrations/mongo/mongo.go index 012a1a0726..af6dd4701a 100644 --- a/pkg/core/proxy/integrations/mongo/mongo.go +++ b/pkg/agent/proxy/integrations/mongo/mongo.go @@ -9,10 +9,10 @@ import ( "sync" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" "go.keploy.io/server/v2/utils" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.mongodb.org/mongo-driver/x/mongo/driver/wiremessage" "go.uber.org/zap" @@ -48,7 +48,7 @@ func (m *Mongo) MatchType(_ context.Context, buffer []byte) bool { // RecordOutgoing records the outgoing mongo messages of the client connection into the yaml file. // The database connection is keep-alive so, this function will be called during the connection establishment. -func (m *Mongo) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (m *Mongo) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := m.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) if err != nil { @@ -61,7 +61,7 @@ func (m *Mongo) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, // initially the reqBuf contains the first network packet // from the client connection which is used to determine // the packet type in MatchType. - err = m.encodeMongo(ctx, logger, reqBuf, src, dst, mocks, opts) + err = m.encodeMongo(ctx, logger, reqBuf, src, dst, mocks, clientClose, opts) if err != nil { utils.LogError(logger, err, "failed to encode the mongo message into the yaml") return err @@ -82,6 +82,7 @@ func (m *Mongo) MockOutgoing(ctx context.Context, src net.Conn, dstCfg *models.C return err } + m.logger.Info("Mocking the mongo message") // converts the yaml string into the binary packet err = decodeMongo(ctx, logger, reqBuf, src, dstCfg, mockDb, opts) if err != nil { diff --git a/pkg/core/proxy/integrations/mongo/operation.go b/pkg/agent/proxy/integrations/mongo/operation.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/operation.go rename to pkg/agent/proxy/integrations/mongo/operation.go index 9a2fcf9315..98aefde79b 100644 --- a/pkg/core/proxy/integrations/mongo/operation.go +++ b/pkg/agent/proxy/integrations/mongo/operation.go @@ -12,8 +12,8 @@ import ( "strings" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/scram" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/scram" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.mongodb.org/mongo-driver/bson" diff --git a/pkg/core/proxy/integrations/mongo/scramAuth.go b/pkg/agent/proxy/integrations/mongo/scramAuth.go similarity index 99% rename from pkg/core/proxy/integrations/mongo/scramAuth.go rename to pkg/agent/proxy/integrations/mongo/scramAuth.go index e75971ce5a..2b90eb56ae 100644 --- a/pkg/core/proxy/integrations/mongo/scramAuth.go +++ b/pkg/agent/proxy/integrations/mongo/scramAuth.go @@ -11,10 +11,10 @@ import ( "strings" "sync" - scramUtil "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + scramUtil "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/scram" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/scram" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mongo/util.go b/pkg/agent/proxy/integrations/mongo/util.go similarity index 100% rename from pkg/core/proxy/integrations/mongo/util.go rename to pkg/agent/proxy/integrations/mongo/util.go diff --git a/pkg/core/proxy/integrations/mysql/README.md b/pkg/agent/proxy/integrations/mysql/README.md similarity index 100% rename from pkg/core/proxy/integrations/mysql/README.md rename to pkg/agent/proxy/integrations/mysql/README.md diff --git a/pkg/core/proxy/integrations/mysql/mysql.go b/pkg/agent/proxy/integrations/mysql/mysql.go similarity index 81% rename from pkg/core/proxy/integrations/mysql/mysql.go rename to pkg/agent/proxy/integrations/mysql/mysql.go index 1914a5262e..b961fdd904 100644 --- a/pkg/core/proxy/integrations/mysql/mysql.go +++ b/pkg/agent/proxy/integrations/mysql/mysql.go @@ -8,9 +8,9 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/recorder" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/replayer" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/recorder" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/replayer" "go.keploy.io/server/v2/utils" @@ -37,10 +37,10 @@ func (m *MySQL) MatchType(_ context.Context, _ []byte) bool { return false } -func (m *MySQL) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (m *MySQL) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := m.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) - err := recorder.Record(ctx, logger, src, dst, mocks, opts) + err := recorder.Record(ctx, logger, src, dst, mocks, opts, clientClose) if err != nil { utils.LogError(logger, err, "failed to encode the mysql message into the yaml") return err diff --git a/pkg/core/proxy/integrations/mysql/recorder/conn.go b/pkg/agent/proxy/integrations/mysql/recorder/conn.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/recorder/conn.go rename to pkg/agent/proxy/integrations/mysql/recorder/conn.go index 89ce66ac34..4a21045ee9 100644 --- a/pkg/core/proxy/integrations/mysql/recorder/conn.go +++ b/pkg/agent/proxy/integrations/mysql/recorder/conn.go @@ -11,11 +11,11 @@ import ( "net" "time" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - intgUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pTls "go.keploy.io/server/v2/pkg/core/proxy/tls" - pUtils "go.keploy.io/server/v2/pkg/core/proxy/util" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + intgUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pTls "go.keploy.io/server/v2/pkg/agent/proxy/tls" + pUtils "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/recorder/query.go b/pkg/agent/proxy/integrations/mysql/recorder/query.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/recorder/query.go rename to pkg/agent/proxy/integrations/mysql/recorder/query.go index 122b57ebc4..9edb22b885 100644 --- a/pkg/core/proxy/integrations/mysql/recorder/query.go +++ b/pkg/agent/proxy/integrations/mysql/recorder/query.go @@ -9,9 +9,9 @@ import ( "net" "time" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/recorder/record.go b/pkg/agent/proxy/integrations/mysql/recorder/record.go similarity index 94% rename from pkg/core/proxy/integrations/mysql/recorder/record.go rename to pkg/agent/proxy/integrations/mysql/recorder/record.go index 3b48d234ac..c14fb8db6b 100644 --- a/pkg/core/proxy/integrations/mysql/recorder/record.go +++ b/pkg/agent/proxy/integrations/mysql/recorder/record.go @@ -12,8 +12,8 @@ import ( "golang.org/x/sync/errgroup" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" @@ -22,7 +22,7 @@ import ( // Binary to Mock Yaml -func Record(ctx context.Context, logger *zap.Logger, clientConn, destConn net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func Record(ctx context.Context, logger *zap.Logger, clientConn, destConn net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions, clientClose chan bool) error { var ( requests []mysql.Request @@ -97,6 +97,9 @@ func Record(ctx context.Context, logger *zap.Logger, clientConn, destConn net.Co }) select { + case <-clientClose: + mocks <- &models.Mock{} + return ctx.Err() case <-ctx.Done(): return ctx.Err() case err := <-errCh: @@ -105,6 +108,7 @@ func Record(ctx context.Context, logger *zap.Logger, clientConn, destConn net.Co } return err } + } func recordMock(_ context.Context, requests []mysql.Request, responses []mysql.Response, mockType, requestOperation, responseOperation string, mocks chan<- *models.Mock, reqTimestampMock time.Time) { diff --git a/pkg/core/proxy/integrations/mysql/replayer/conn.go b/pkg/agent/proxy/integrations/mysql/replayer/conn.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/replayer/conn.go rename to pkg/agent/proxy/integrations/mysql/replayer/conn.go index ea7d09ef50..269bf73186 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/conn.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/conn.go @@ -9,11 +9,11 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - pTls "go.keploy.io/server/v2/pkg/core/proxy/tls" - pUtils "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + pTls "go.keploy.io/server/v2/pkg/agent/proxy/tls" + pUtils "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/match.go b/pkg/agent/proxy/integrations/mysql/replayer/match.go similarity index 99% rename from pkg/core/proxy/integrations/mysql/replayer/match.go rename to pkg/agent/proxy/integrations/mysql/replayer/match.go index 3cc2a335c6..ad5681a298 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/match.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/match.go @@ -9,9 +9,9 @@ import ( "io" "math" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - intgUtil "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + intgUtil "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/query.go b/pkg/agent/proxy/integrations/mysql/replayer/query.go similarity index 94% rename from pkg/core/proxy/integrations/mysql/replayer/query.go rename to pkg/agent/proxy/integrations/mysql/replayer/query.go index 4268b968ec..4a03e8494b 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/query.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/query.go @@ -9,9 +9,9 @@ import ( "net" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/replayer/replay.go b/pkg/agent/proxy/integrations/mysql/replayer/replay.go similarity index 92% rename from pkg/core/proxy/integrations/mysql/replayer/replay.go rename to pkg/agent/proxy/integrations/mysql/replayer/replay.go index 106b0f3812..e66d9e1bea 100644 --- a/pkg/core/proxy/integrations/mysql/replayer/replay.go +++ b/pkg/agent/proxy/integrations/mysql/replayer/replay.go @@ -8,10 +8,10 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire" - intgUtil "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire" + intgUtil "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" diff --git a/pkg/core/proxy/integrations/mysql/utils/util.go b/pkg/agent/proxy/integrations/mysql/utils/util.go similarity index 99% rename from pkg/core/proxy/integrations/mysql/utils/util.go rename to pkg/agent/proxy/integrations/mysql/utils/util.go index f241a4bcca..a3e8e19739 100644 --- a/pkg/core/proxy/integrations/mysql/utils/util.go +++ b/pkg/agent/proxy/integrations/mysql/utils/util.go @@ -11,7 +11,7 @@ import ( "io" "net" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models/mysql" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/mysql/wire/decode.go b/pkg/agent/proxy/integrations/mysql/wire/decode.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/decode.go rename to pkg/agent/proxy/integrations/mysql/wire/decode.go index 40306b515a..8a5c2b3b4c 100644 --- a/pkg/core/proxy/integrations/mysql/wire/decode.go +++ b/pkg/agent/proxy/integrations/mysql/wire/decode.go @@ -8,14 +8,14 @@ import ( "fmt" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase" - connection "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/conn" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/utility" - - itgUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase" + connection "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/conn" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/utility" + + itgUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/mysql/wire/encode.go b/pkg/agent/proxy/integrations/mysql/wire/encode.go similarity index 93% rename from pkg/core/proxy/integrations/mysql/wire/encode.go rename to pkg/agent/proxy/integrations/mysql/wire/encode.go index 566745cc13..286a5dc39b 100644 --- a/pkg/core/proxy/integrations/mysql/wire/encode.go +++ b/pkg/agent/proxy/integrations/mysql/wire/encode.go @@ -8,10 +8,10 @@ import ( "fmt" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/conn" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/conn" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authMoreDataPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go similarity index 95% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go index ddeb0652b5..5950cd48ed 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authNextFactorPacket.go @@ -8,7 +8,7 @@ import ( "errors" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchRequestPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/authSwitchResponsePacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go index 73a8c15a58..93226b9857 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go @@ -9,7 +9,8 @@ import ( "errors" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) @@ -84,7 +85,7 @@ func DecodeHandshakeResponse(_ context.Context, logger *zap.Logger, data []byte) if packet.CapabilityFlags&mysql.CLIENT_CONNECT_WITH_DB != 0 { idx = bytes.IndexByte(data, 0x00) if idx != -1 { - packet.Database = string(data[:idx]) + packet.Database = util.EncodeBase64(data[:idx]) data = data[idx+1:] } } @@ -94,7 +95,7 @@ func DecodeHandshakeResponse(_ context.Context, logger *zap.Logger, data []byte) if idx == -1 { return nil, errors.New("malformed handshake response packet: missing null terminator for AuthPluginName") } - packet.AuthPluginName = string(data[:idx]) + packet.AuthPluginName = util.EncodeBase64(data[:idx]) data = data[idx+1:] } diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go b/pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/conn/handshakeV10Packet.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/generic.go b/pkg/agent/proxy/integrations/mysql/wire/phase/generic.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/wire/phase/generic.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/generic.go index d0663b76fe..e7d9fa2027 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/generic.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/generic.go @@ -9,7 +9,7 @@ import ( "encoding/binary" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go similarity index 94% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go index 672b09cc26..ebb11cc76c 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/ResultSetPacket.go @@ -7,9 +7,9 @@ import ( "context" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - mysqlUtils "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + mysqlUtils "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go index b7864663e1..6f9037b967 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/StmtPrepareOkPacket.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtClosePacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go similarity index 97% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go index db53c6576d..02b6e43510 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtExecutePacket.go @@ -8,7 +8,7 @@ import ( "fmt" "io" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtFetchPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtPreparePacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtResetPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/preparedstmt/stmtSendLongDataPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/queryPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/queryPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/queryPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/queryPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go similarity index 82% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go index a985fc62ac..30141d8c4e 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go @@ -11,7 +11,7 @@ import ( "fmt" "strings" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) @@ -255,14 +255,37 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow columnEntry := row.Values[i] + fmt.Printf("ColumnEntry: %+v\n", columnEntry) + fmt.Printf("columnEntry.Value: %+v\n", columnEntry.Value) + fmt.Printf("Type of Value: %T\n", columnEntry.Value) + switch columnEntry.Type { case mysql.FieldTypeLong: var val any if columnEntry.Unsigned { - val = uint32(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint32(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint32(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint32(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint32(columnEntry.Value.(uint)) + } } else { - val = int32(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int32(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int32(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int32(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int32(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } @@ -277,10 +300,29 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow case mysql.FieldTypeTiny: var val any if columnEntry.Unsigned { - val = uint8(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint8(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint8(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint8(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint8(columnEntry.Value.(uint)) + } } else { - val = int8(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int8(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int8(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int8(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int8(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } @@ -288,21 +330,60 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow case mysql.FieldTypeShort, mysql.FieldTypeYear: var val any if columnEntry.Unsigned { - val = uint16(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint16(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint16(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint16(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint16(columnEntry.Value.(uint)) + } + } else { - val = int16(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int16(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int16(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int16(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int16(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write int16 value: %w", err) } case mysql.FieldTypeLongLong: var val any if columnEntry.Unsigned { - val = uint64(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint64(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint64(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint64(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint64(columnEntry.Value.(uint)) + } } else { - val = int64(columnEntry.Value.(int)) - } + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int64(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int64(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int64(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int64(columnEntry.Value.(uint)) + } + } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go similarity index 89% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go index 3d16dbefa5..a3da90dde0 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnCountPacket.go @@ -6,7 +6,7 @@ import ( "context" "errors" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go index db554015b0..65eddcd317 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/columnPacket.go @@ -8,7 +8,7 @@ import ( "encoding/binary" "fmt" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go similarity index 98% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go index 6c50ebfdc1..294959d0da 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go +++ b/pkg/agent/proxy/integrations/mysql/wire/phase/query/rowscols/textRowPacket.go @@ -8,7 +8,7 @@ import ( "fmt" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql/utils" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/initDbPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go b/pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go rename to pkg/agent/proxy/integrations/mysql/wire/phase/query/utility/setOptionPacket.go diff --git a/pkg/core/proxy/integrations/mysql/wire/util.go b/pkg/agent/proxy/integrations/mysql/wire/util.go similarity index 100% rename from pkg/core/proxy/integrations/mysql/wire/util.go rename to pkg/agent/proxy/integrations/mysql/wire/util.go diff --git a/pkg/core/proxy/integrations/postgres/v1/decode.go b/pkg/agent/proxy/integrations/postgres/v1/decode.go similarity index 95% rename from pkg/core/proxy/integrations/postgres/v1/decode.go rename to pkg/agent/proxy/integrations/postgres/v1/decode.go index 308f2625bb..b49e171987 100644 --- a/pkg/core/proxy/integrations/postgres/v1/decode.go +++ b/pkg/agent/proxy/integrations/postgres/v1/decode.go @@ -12,9 +12,9 @@ import ( "sync" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -67,7 +67,7 @@ func decodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie } if !matched { - logger.Debug("MISMATCHED REQ is" + string(pgRequests[0])) + logger.Info("MISMATCHED REQ is" + string(pgRequests[0])) _, err = pUtil.PassThrough(ctx, logger, clientConn, dstCfg, pgRequests) if err != nil { utils.LogError(logger, err, "failed to pass the request", zap.Any("request packets", len(pgRequests))) diff --git a/pkg/core/proxy/integrations/postgres/v1/encode.go b/pkg/agent/proxy/integrations/postgres/v1/encode.go similarity index 85% rename from pkg/core/proxy/integrations/postgres/v1/encode.go rename to pkg/agent/proxy/integrations/postgres/v1/encode.go index 90b0d739d6..af565da796 100755 --- a/pkg/core/proxy/integrations/postgres/v1/encode.go +++ b/pkg/agent/proxy/integrations/postgres/v1/encode.go @@ -12,15 +12,15 @@ import ( "time" "github.com/jackc/pgproto3/v2" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) -func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, clientClose chan bool, _ models.OutgoingOptions) error { logger.Debug("Inside the encodePostgresOutgoing function") var pgRequests []models.Backend @@ -112,6 +112,28 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie for { select { + case <-clientClose: + logger.Info("client connection is closed") + if !prevChunkWasReq && len(pgRequests) > 0 && len(pgResponses) > 0 { + metadata := make(map[string]string) + metadata["type"] = "config" + // Save the mock + m := &models.Mock{ + Version: models.GetVersion(), + Name: "mocks", + Kind: models.Postgres, + Spec: models.MockSpec{ + PostgresRequests: pgRequests, + PostgresResponses: pgResponses, + ReqTimestampMock: reqTimestampMock, + ResTimestampMock: resTimestampMock, + Metadata: metadata, + }, + ConnectionID: ctx.Value(models.ClientConnectionIDKey).(string), + } + mocks <- m + return ctx.Err() + } case <-ctx.Done(): if !prevChunkWasReq && len(pgRequests) > 0 && len(pgResponses) > 0 { metadata := make(map[string]string) @@ -130,6 +152,7 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie }, ConnectionID: ctx.Value(models.ClientConnectionIDKey).(string), } + fmt.Println("Context is done in the postgres encode function", mocks) return ctx.Err() } case buffer := <-clientBuffChan: @@ -326,10 +349,9 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie // from here take the msg and append its readable form to the pgResponses pgMock := &models.Frontend{ - PacketTypes: pg.FrontendWrapper.PacketTypes, - Identfier: "ServerResponse", - Length: uint32(len(reqBuf)), - // Payload: bufStr, + PacketTypes: pg.FrontendWrapper.PacketTypes, + Identfier: "ServerResponse", + Length: uint32(len(reqBuf)), AuthenticationOk: pg.FrontendWrapper.AuthenticationOk, AuthenticationCleartextPassword: pg.FrontendWrapper.AuthenticationCleartextPassword, AuthenticationMD5Password: pg.FrontendWrapper.AuthenticationMD5Password, @@ -345,24 +367,24 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie CommandCompletes: pg.FrontendWrapper.CommandCompletes, CopyData: pg.FrontendWrapper.CopyData, CopyDone: pg.FrontendWrapper.CopyDone, - CopyInResponse: pg.FrontendWrapper.CopyInResponse, - CopyOutResponse: pg.FrontendWrapper.CopyOutResponse, - DataRow: pg.FrontendWrapper.DataRow, - DataRows: pg.FrontendWrapper.DataRows, - EmptyQueryResponse: pg.FrontendWrapper.EmptyQueryResponse, - ErrorResponse: pg.FrontendWrapper.ErrorResponse, - FunctionCallResponse: pg.FrontendWrapper.FunctionCallResponse, - NoData: pg.FrontendWrapper.NoData, - NoticeResponse: pg.FrontendWrapper.NoticeResponse, - NotificationResponse: pg.FrontendWrapper.NotificationResponse, - ParameterDescription: pg.FrontendWrapper.ParameterDescription, - ParameterStatusCombined: pg.FrontendWrapper.ParameterStatusCombined, - ParseComplete: pg.FrontendWrapper.ParseComplete, - PortalSuspended: pg.FrontendWrapper.PortalSuspended, - ReadyForQuery: pg.FrontendWrapper.ReadyForQuery, - RowDescription: pg.FrontendWrapper.RowDescription, - MsgType: pg.FrontendWrapper.MsgType, - AuthType: pg.FrontendWrapper.AuthType, + // CopyInResponse: pg.FrontendWrapper.CopyInResponse, + // CopyOutResponse: pg.FrontendWrapper.CopyOutResponse, + DataRow: pg.FrontendWrapper.DataRow, + DataRows: pg.FrontendWrapper.DataRows, + EmptyQueryResponse: pg.FrontendWrapper.EmptyQueryResponse, + ErrorResponse: pg.FrontendWrapper.ErrorResponse, + FunctionCallResponse: pg.FrontendWrapper.FunctionCallResponse, + NoData: pg.FrontendWrapper.NoData, + NoticeResponse: pg.FrontendWrapper.NoticeResponse, + NotificationResponse: pg.FrontendWrapper.NotificationResponse, + ParameterDescription: pg.FrontendWrapper.ParameterDescription, + ParameterStatusCombined: pg.FrontendWrapper.ParameterStatusCombined, + ParseComplete: pg.FrontendWrapper.ParseComplete, + PortalSuspended: pg.FrontendWrapper.PortalSuspended, + ReadyForQuery: pg.FrontendWrapper.ReadyForQuery, + RowDescription: pg.FrontendWrapper.RowDescription, + MsgType: pg.FrontendWrapper.MsgType, + AuthType: pg.FrontendWrapper.AuthType, } afterEncoded, err := postgresDecoderFrontend(*pgMock) diff --git a/pkg/core/proxy/integrations/postgres/v1/match.go b/pkg/agent/proxy/integrations/postgres/v1/match.go similarity index 98% rename from pkg/core/proxy/integrations/postgres/v1/match.go rename to pkg/agent/proxy/integrations/postgres/v1/match.go index 819a9bd35d..b2b219414c 100644 --- a/pkg/core/proxy/integrations/postgres/v1/match.go +++ b/pkg/agent/proxy/integrations/postgres/v1/match.go @@ -14,8 +14,8 @@ import ( "github.com/agnivade/levenshtein" "github.com/jackc/pgproto3/v2" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.uber.org/zap" ) @@ -93,7 +93,6 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut reqGoingOn := decodePgRequest(requestBuffers[0], logger) if reqGoingOn != nil { logger.Debug("PacketTypes", zap.Any("PacketTypes", reqGoingOn.PacketTypes)) - // fmt.Println("REQUEST GOING ON - ", reqGoingOn) logger.Debug("ConnectionId-", zap.String("ConnectionId", ConnectionID)) logger.Debug("TestMap*****", zap.Any("TestMap", testmap)) } @@ -142,7 +141,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut } return true, []models.Frontend{ssl}, nil case initMock.Spec.PostgresRequests[requestIndex].Identfier == "StartupRequest" && isStartupPacket(reqBuff) && initMock.Spec.PostgresRequests[requestIndex].Payload != "AAAACATSFi8=" && initMock.Spec.PostgresResponses[requestIndex].AuthType == 10: - logger.Debug("CHANGING TO MD5 for Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) + logger.Info("CHANGING TO MD5 for Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) res := make([]models.Frontend, len(initMock.Spec.PostgresResponses)) copy(res, initMock.Spec.PostgresResponses) res[requestIndex].AuthType = 5 @@ -480,7 +479,7 @@ func changeResToPS(mock *models.Mock, actualPgReq *models.Backend, logger *zap.L mockPackets := mock.Spec.PostgresRequests[0].PacketTypes // [P, B, E, P, B, D, E] => [B, E, B, E] - // write code that of packet is ["B", "E"] and mockPackets ["P", "B", "D", "E"] handle it in case1 + // packet is ["B", "E"] and mockPackets ["P", "B", "D", "E"] handle it in case1 // and if packet is [B, E, B, E] and mockPackets [P, B, E, P, B, D, E] handle it in case2 ischanged := false @@ -534,7 +533,6 @@ func changeResToPS(mock *models.Mock, actualPgReq *models.Backend, logger *zap.L break } } - //Matched In Binary Matching for Unsorted mock-222 ischanged2 := false ps2 := actualPgReq.Binds[1].PreparedStatement for _, v := range testmap[connectionID] { @@ -551,7 +549,6 @@ func changeResToPS(mock *models.Mock, actualPgReq *models.Backend, logger *zap.L // Case 4 if reflect.DeepEqual(actualpackets, []string{"B", "E", "B", "E"}) && reflect.DeepEqual(mockPackets, []string{"B", "E", "P", "B", "D", "E"}) { - // logger.Debug("Handling Case 4 for mock", mock.Name) // get the query for the prepared statement of test mode ischanged := false ps := actualPgReq.Binds[1].PreparedStatement diff --git a/pkg/core/proxy/integrations/postgres/v1/postgres.go b/pkg/agent/proxy/integrations/postgres/v1/postgres.go similarity index 89% rename from pkg/core/proxy/integrations/postgres/v1/postgres.go rename to pkg/agent/proxy/integrations/postgres/v1/postgres.go index a4eecaaaaf..1b7ddabdf4 100755 --- a/pkg/core/proxy/integrations/postgres/v1/postgres.go +++ b/pkg/agent/proxy/integrations/postgres/v1/postgres.go @@ -7,8 +7,8 @@ import ( "encoding/binary" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/pkg/models" @@ -49,7 +49,7 @@ func (p *PostgresV1) MatchType(_ context.Context, reqBuf []byte) bool { return version == ProtocolVersion } -func (p *PostgresV1) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (p *PostgresV1) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := p.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) @@ -57,7 +57,7 @@ func (p *PostgresV1) RecordOutgoing(ctx context.Context, src net.Conn, dst net.C utils.LogError(logger, err, "failed to read the initial postgres message") return err } - err = encodePostgres(ctx, logger, reqBuf, src, dst, mocks, opts) + err = encodePostgres(ctx, logger, reqBuf, src, dst, mocks, clientClose, opts) if err != nil { // TODO: why debug log? logger.Debug("failed to encode the postgres message into the yaml") diff --git a/pkg/core/proxy/integrations/postgres/v1/transcoder.go b/pkg/agent/proxy/integrations/postgres/v1/transcoder.go similarity index 97% rename from pkg/core/proxy/integrations/postgres/v1/transcoder.go rename to pkg/agent/proxy/integrations/postgres/v1/transcoder.go index 9e1ce08c9a..3f2d7059fb 100644 --- a/pkg/core/proxy/integrations/postgres/v1/transcoder.go +++ b/pkg/agent/proxy/integrations/postgres/v1/transcoder.go @@ -30,7 +30,6 @@ func NewFrontend() *FrontendWrapper { // PG Response Packet Transcoder func (b *BackendWrapper) translateToReadableBackend(msgBody []byte) (pgproto3.FrontendMessage, error) { - // fmt.Println("msgType", b.BackendWrapper.MsgType) var msg pgproto3.FrontendMessage switch b.BackendWrapper.MsgType { case 'B': @@ -111,9 +110,9 @@ func (f *FrontendWrapper) translateToReadableResponse(logger *zap.Logger, msgBod case 'E': msg = &f.FrontendWrapper.ErrorResponse case 'G': - msg = &f.FrontendWrapper.CopyInResponse - case 'H': - msg = &f.FrontendWrapper.CopyOutResponse + // msg = &f.FrontendWrapper.CopyInResponse + // case 'H': + // msg = &f.FrontendWrapper.CopyOutResponse case 'I': msg = &f.FrontendWrapper.EmptyQueryResponse case 'K': @@ -138,8 +137,6 @@ func (f *FrontendWrapper) translateToReadableResponse(logger *zap.Logger, msgBod msg = &f.FrontendWrapper.RowDescription case 'V': msg = &f.FrontendWrapper.FunctionCallResponse - case 'W': - msg = &f.FrontendWrapper.CopyBothResponse case 'Z': msg = &f.FrontendWrapper.ReadyForQuery default: diff --git a/pkg/core/proxy/integrations/postgres/v1/util.go b/pkg/agent/proxy/integrations/postgres/v1/util.go similarity index 94% rename from pkg/core/proxy/integrations/postgres/v1/util.go rename to pkg/agent/proxy/integrations/postgres/v1/util.go index a9d3faf34d..2b1df9620e 100755 --- a/pkg/core/proxy/integrations/postgres/v1/util.go +++ b/pkg/agent/proxy/integrations/postgres/v1/util.go @@ -10,7 +10,7 @@ import ( "time" "github.com/jackc/pgproto3/v2" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" "go.uber.org/zap" ) @@ -78,16 +78,16 @@ func postgresDecoderFrontend(response models.Frontend) ([]byte, error) { Line: response.ErrorResponse.Line, Routine: response.ErrorResponse.Routine, } - case string('G'): - msg = &pgproto3.CopyInResponse{ - OverallFormat: response.CopyInResponse.OverallFormat, - ColumnFormatCodes: response.CopyInResponse.ColumnFormatCodes, - } - case string('H'): - msg = &pgproto3.CopyOutResponse{ - OverallFormat: response.CopyOutResponse.OverallFormat, - ColumnFormatCodes: response.CopyOutResponse.ColumnFormatCodes, - } + // case string('G'): + // msg = &pgproto3.CopyInResponse{ + // OverallFormat: response.CopyInResponse.OverallFormat, + // ColumnFormatCodes: response.CopyInResponse.ColumnFormatCodes, + // } + // case string('H'): + // msg = &pgproto3.CopyOutResponse{ + // OverallFormat: response.CopyOutResponse.OverallFormat, + // ColumnFormatCodes: response.CopyOutResponse.ColumnFormatCodes, + // } case string('I'): msg = &pgproto3.EmptyQueryResponse{} case string('K'): @@ -165,11 +165,11 @@ func postgresDecoderFrontend(response models.Frontend) ([]byte, error) { msg = &pgproto3.FunctionCallResponse{ Result: response.FunctionCallResponse.Result, } - case string('W'): - msg = &pgproto3.CopyBothResponse{ - OverallFormat: response.CopyBothResponse.OverallFormat, - ColumnFormatCodes: response.CopyBothResponse.ColumnFormatCodes, - } + // case string('W'): + // msg = &pgproto3.CopyBothResponse{ + // // OverallFormat: response.CopyBothResponse.OverallFormat, + // ColumnFormatCodes: response.CopyBothResponse.ColumnFormatCodes, + // } case string('Z'): msg = &pgproto3.ReadyForQuery{ TxStatus: response.ReadyForQuery.TxStatus, @@ -320,7 +320,6 @@ func sliceCommandTag(mock *models.Mock, logger *zap.Logger, prep []QueryData, ac case 1: copyMock := *mock - // fmt.Println("Inside Slice Command Tag for ", psCase) mockPackets := copyMock.Spec.PostgresResponses[0].PacketTypes for idx, v := range mockPackets { if v == "1" { @@ -332,9 +331,7 @@ func sliceCommandTag(mock *models.Mock, logger *zap.Logger, prep []QueryData, ac return ©Mock case 2: - // ["2", D, C, Z] copyMock := *mock - // fmt.Println("Inside Slice Command Tag for ", psCase) mockPackets := copyMock.Spec.PostgresResponses[0].PacketTypes for idx, v := range mockPackets { if v == "1" || v == "T" { @@ -347,11 +344,8 @@ func sliceCommandTag(mock *models.Mock, logger *zap.Logger, prep []QueryData, ac for idx, datarow := range copyMock.Spec.PostgresResponses[0].DataRows { for column, rowVal := range datarow.RowValues { - // fmt.Println("datarow.RowValues", len(datarow.RowValues)) if rsFormat[column] == 1 { - // datarows := make([]byte, 4) newRow, _ := getChandedDataRow(rowVal) - // logger.Info("New Row Value", zap.String("newRow", newRow)) copyMock.Spec.PostgresResponses[0].DataRows[idx].RowValues[column] = newRow } } diff --git a/pkg/core/proxy/integrations/redis/decode.go b/pkg/agent/proxy/integrations/redis/decode.go similarity index 95% rename from pkg/core/proxy/integrations/redis/decode.go rename to pkg/agent/proxy/integrations/redis/decode.go index 4bb83d31b2..6d5a4953f7 100644 --- a/pkg/core/proxy/integrations/redis/decode.go +++ b/pkg/agent/proxy/integrations/redis/decode.go @@ -9,9 +9,9 @@ import ( "net" "time" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/integrations/redis/encode.go b/pkg/agent/proxy/integrations/redis/encode.go similarity index 96% rename from pkg/core/proxy/integrations/redis/encode.go rename to pkg/agent/proxy/integrations/redis/encode.go index 7d34d7ce14..4bcf71f27a 100644 --- a/pkg/core/proxy/integrations/redis/encode.go +++ b/pkg/agent/proxy/integrations/redis/encode.go @@ -11,13 +11,13 @@ import ( "golang.org/x/sync/errgroup" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) -func encodeRedis(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions) error { +func encodeRedis(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientConn, destConn net.Conn, mocks chan<- *models.Mock, _ models.OutgoingOptions, clientClose chan bool) error { var redisRequests []models.Payload var redisResponses []models.Payload @@ -136,6 +136,9 @@ func encodeRedis(ctx context.Context, logger *zap.Logger, reqBuf []byte, clientC }) select { + case <-clientClose: + mocks <- &models.Mock{} + return ctx.Err() case <-ctx.Done(): return ctx.Err() case err := <-errCh: diff --git a/pkg/core/proxy/integrations/redis/match.go b/pkg/agent/proxy/integrations/redis/match.go similarity index 97% rename from pkg/core/proxy/integrations/redis/match.go rename to pkg/agent/proxy/integrations/redis/match.go index 9e3539067c..28debea482 100755 --- a/pkg/core/proxy/integrations/redis/match.go +++ b/pkg/agent/proxy/integrations/redis/match.go @@ -7,9 +7,9 @@ import ( "fmt" "math" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models" ) diff --git a/pkg/core/proxy/integrations/redis/redis.go b/pkg/agent/proxy/integrations/redis/redis.go similarity index 87% rename from pkg/core/proxy/integrations/redis/redis.go rename to pkg/agent/proxy/integrations/redis/redis.go index 0a3e7521ca..4af0422755 100755 --- a/pkg/core/proxy/integrations/redis/redis.go +++ b/pkg/agent/proxy/integrations/redis/redis.go @@ -6,8 +6,8 @@ import ( "context" "net" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -41,7 +41,7 @@ func (r *Redis) MatchType(_ context.Context, buf []byte) bool { } } -func (r *Redis) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { +func (r *Redis) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, mocks chan<- *models.Mock, clientClose chan bool, opts models.OutgoingOptions) error { logger := r.logger.With(zap.Any("Client IP Address", src.RemoteAddr().String()), zap.Any("Client ConnectionID", ctx.Value(models.ClientConnectionIDKey).(string)), zap.Any("Destination ConnectionID", ctx.Value(models.DestConnectionIDKey).(string))) reqBuf, err := util.ReadInitialBuf(ctx, logger, src) @@ -50,7 +50,7 @@ func (r *Redis) RecordOutgoing(ctx context.Context, src net.Conn, dst net.Conn, return err } - err = encodeRedis(ctx, logger, reqBuf, src, dst, mocks, opts) + err = encodeRedis(ctx, logger, reqBuf, src, dst, mocks, opts, clientClose) if err != nil { utils.LogError(logger, err, "failed to encode the redis message into the yaml") return err diff --git a/pkg/core/proxy/integrations/scram/scram.go b/pkg/agent/proxy/integrations/scram/scram.go similarity index 99% rename from pkg/core/proxy/integrations/scram/scram.go rename to pkg/agent/proxy/integrations/scram/scram.go index 4b923c565c..8525260d24 100644 --- a/pkg/core/proxy/integrations/scram/scram.go +++ b/pkg/agent/proxy/integrations/scram/scram.go @@ -12,7 +12,7 @@ import ( "github.com/xdg-go/pbkdf2" "github.com/xdg-go/scram" "github.com/xdg-go/stringprep" - "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations/util" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/integrations/scram/util.go b/pkg/agent/proxy/integrations/scram/util.go similarity index 100% rename from pkg/core/proxy/integrations/scram/util.go rename to pkg/agent/proxy/integrations/scram/util.go diff --git a/pkg/core/proxy/integrations/util/util.go b/pkg/agent/proxy/integrations/util/util.go similarity index 100% rename from pkg/core/proxy/integrations/util/util.go rename to pkg/agent/proxy/integrations/util/util.go diff --git a/pkg/core/proxy/mockmanager.go b/pkg/agent/proxy/mockmanager.go similarity index 100% rename from pkg/core/proxy/mockmanager.go rename to pkg/agent/proxy/mockmanager.go diff --git a/pkg/core/proxy/options.go b/pkg/agent/proxy/options.go similarity index 100% rename from pkg/core/proxy/options.go rename to pkg/agent/proxy/options.go diff --git a/pkg/agent/proxy/parsers.go b/pkg/agent/proxy/parsers.go new file mode 100644 index 0000000000..26ba078f33 --- /dev/null +++ b/pkg/agent/proxy/parsers.go @@ -0,0 +1,14 @@ +//go:build linux + +package proxy + +import ( + // import all the integrations + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/generic" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/grpc" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/http" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mongo" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/mysql" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/postgres/v1" + _ "go.keploy.io/server/v2/pkg/agent/proxy/integrations/redis" +) diff --git a/pkg/core/proxy/proxy.go b/pkg/agent/proxy/proxy.go similarity index 90% rename from pkg/core/proxy/proxy.go rename to pkg/agent/proxy/proxy.go index 2e60be8326..d59faa20ca 100755 --- a/pkg/core/proxy/proxy.go +++ b/pkg/agent/proxy/proxy.go @@ -21,11 +21,11 @@ import ( "github.com/miekg/dns" "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/proxy/integrations" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/proxy/integrations" - pTls "go.keploy.io/server/v2/pkg/core/proxy/tls" - "go.keploy.io/server/v2/pkg/core/proxy/util" + pTls "go.keploy.io/server/v2/pkg/agent/proxy/tls" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -39,19 +39,21 @@ type Proxy struct { Port uint32 DNSPort uint32 - DestInfo core.DestInfo + DestInfo agent.DestInfo Integrations map[string]integrations.Integrations MockManagers sync.Map - sessions *core.Sessions + sessions *agent.Sessions connMutex *sync.Mutex ipMutex *sync.Mutex clientConnections []net.Conn - Listener net.Listener + // channel to mark client connection as closed + clientClose chan bool + Listener net.Listener //to store the nsswitch.conf file data nsswitchData []byte // in test mode we change the configuration of "hosts" in nsswitch.conf file to disable resolution over unix socket @@ -59,17 +61,19 @@ type Proxy struct { TCPDNSServer *dns.Server } -func New(logger *zap.Logger, info core.DestInfo, opts *config.Config) *Proxy { +func New(logger *zap.Logger, info agent.DestInfo, opts *config.Config) *Proxy { + return &Proxy{ logger: logger, - Port: opts.ProxyPort, // default: 16789 - DNSPort: opts.DNSPort, // default: 26789 - IP4: "127.0.0.1", // default: "127.0.0.1" <-> (2130706433) - IP6: "::1", //default: "::1" <-> ([4]uint32{0000, 0000, 0000, 0001}) + Port: opts.Agent.ProxyPort, // default: 16789 + DNSPort: opts.DNSPort, // default: 26789 + IP4: "127.0.0.1", // default: "127.0.0.1" <-> (2130706433) + IP6: "::1", //default: "::1" <-> ([4]uint32{0000, 0000, 0000, 0001}) ipMutex: &sync.Mutex{}, connMutex: &sync.Mutex{}, DestInfo: info, - sessions: core.NewSessions(), + sessions: agent.NewSessions(), // sessions to store the session rules + clientClose: make(chan bool), MockManagers: sync.Map{}, Integrations: make(map[string]integrations.Integrations), } @@ -84,7 +88,7 @@ func (p *Proxy) InitIntegrations(_ context.Context) error { return nil } -func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { +func (p *Proxy) StartProxy(ctx context.Context, opts agent.ProxyOptions) error { //first initialize the integrations err := p.InitIntegrations(ctx) @@ -92,7 +96,6 @@ func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { utils.LogError(p.logger, err, "failed to initialize the integrations") return err } - // set up the CA for tls connections err = pTls.SetupCA(ctx, p.logger) if err != nil { @@ -200,6 +203,7 @@ func (p *Proxy) start(ctx context.Context, readyChan chan<- error) error { readyChan <- err return err } + p.Listener = listener p.logger.Debug(fmt.Sprintf("Proxy server is listening on %s", fmt.Sprintf(":%v", listener.Addr()))) // Signal that the server is ready @@ -219,7 +223,7 @@ func (p *Proxy) start(ctx context.Context, readyChan chan<- error) error { clientConnCancel() err := clientConnErrGrp.Wait() if err != nil { - p.logger.Debug("failed to handle the client connection", zap.Error(err)) + p.logger.Info("failed to handle the client connection", zap.Error(err)) } //closing all the mock channels (if any in record mode) for _, mc := range p.sessions.GetAllMC() { @@ -294,8 +298,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { remoteAddr := srcConn.RemoteAddr().(*net.TCPAddr) sourcePort := remoteAddr.Port - p.logger.Debug("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) - + p.logger.Info("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) destInfo, err := p.DestInfo.Get(ctx, uint16(sourcePort)) if err != nil { utils.LogError(p.logger, err, "failed to fetch the destination info", zap.Any("Source port", sourcePort)) @@ -310,9 +313,10 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { } //get the session rule - rule, ok := p.sessions.Get(destInfo.AppID) + fmt.Println("destInfo.ClientID:::", destInfo.ClientID) + rule, ok := p.sessions.Get(destInfo.ClientID) if !ok { - utils.LogError(p.logger, nil, "failed to fetch the session rule", zap.Any("AppID", destInfo.AppID)) + utils.LogError(p.logger, nil, "failed to fetch the session rule", zap.Any("AppID", destInfo.ClientID)) return err } @@ -391,7 +395,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { rule.OutgoingOptions.DstCfg = dstCfg // Record the outgoing message into a mock - err := p.Integrations["mysql"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, rule.OutgoingOptions) + err := p.Integrations["mysql"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, p.clientClose, rule.OutgoingOptions) if err != nil { utils.LogError(p.logger, err, "failed to record the outgoing message") return err @@ -399,9 +403,9 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { return nil } - m, ok := p.MockManagers.Load(destInfo.AppID) + m, ok := p.MockManagers.Load(destInfo.ClientID) if !ok { - utils.LogError(p.logger, nil, "failed to fetch the mock manager", zap.Any("AppID", destInfo.AppID)) + utils.LogError(p.logger, nil, "failed to fetch the mock manager", zap.Any("AppID", destInfo.ClientID)) return err } @@ -505,9 +509,9 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { } // get the mock manager for the current app - m, ok := p.MockManagers.Load(destInfo.AppID) + m, ok := p.MockManagers.Load(destInfo.ClientID) if !ok { - utils.LogError(logger, err, "failed to fetch the mock manager", zap.Any("AppID", destInfo.AppID)) + utils.LogError(logger, err, "failed to fetch the mock manager", zap.Any("ClientID", destInfo.ClientID)) return err } @@ -517,7 +521,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { for _, parser := range p.Integrations { if parser.MatchType(parserCtx, initialBuf) { if rule.Mode == models.MODE_RECORD { - err := parser.RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, rule.OutgoingOptions) + err := parser.RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, p.clientClose, rule.OutgoingOptions) if err != nil { utils.LogError(logger, err, "failed to record the outgoing message") return err @@ -536,7 +540,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { if generic { logger.Debug("The external dependency is not supported. Hence using generic parser") if rule.Mode == models.MODE_RECORD { - err := p.Integrations["generic"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, rule.OutgoingOptions) + err := p.Integrations["generic"].RecordOutgoing(parserCtx, srcConn, dstConn, rule.MC, p.clientClose, rule.OutgoingOptions) if err != nil { utils.LogError(logger, err, "failed to record the outgoing message") return err @@ -583,14 +587,20 @@ func (p *Proxy) StopProxyServer(ctx context.Context) { p.logger.Info("proxy stopped...") } +func (p *Proxy) MakeClientDeRegisterd(_ context.Context) error { + p.logger.Info("Inside MakeClientDeregisterd of proxyServer") + p.clientClose <- true + return nil +} + func (p *Proxy) Record(_ context.Context, id uint64, mocks chan<- *models.Mock, opts models.OutgoingOptions) error { - p.sessions.Set(id, &core.Session{ + fmt.Println("Inside Record of proxyServer", id) + p.sessions.Set(id, &agent.Session{ ID: id, Mode: models.MODE_RECORD, MC: mocks, OutgoingOptions: opts, }) - p.MockManagers.Store(id, NewMockManager(NewTreeDb(customComparator), NewTreeDb(customComparator), p.logger)) ////set the new proxy ip:port for a new session @@ -603,7 +613,7 @@ func (p *Proxy) Record(_ context.Context, id uint64, mocks chan<- *models.Mock, } func (p *Proxy) Mock(_ context.Context, id uint64, opts models.OutgoingOptions) error { - p.sessions.Set(id, &core.Session{ + p.sessions.Set(id, &agent.Session{ ID: id, Mode: models.MODE_TEST, OutgoingOptions: opts, @@ -642,7 +652,6 @@ func (p *Proxy) SetMocks(_ context.Context, id uint64, filtered []*models.Mock, m.(*MockManager).SetFilteredMocks(filtered) m.(*MockManager).SetUnFilteredMocks(unFiltered) } - return nil } diff --git a/pkg/core/proxy/tls/asset/ca.crt b/pkg/agent/proxy/tls/asset/ca.crt similarity index 100% rename from pkg/core/proxy/tls/asset/ca.crt rename to pkg/agent/proxy/tls/asset/ca.crt diff --git a/pkg/core/proxy/tls/asset/ca.key b/pkg/agent/proxy/tls/asset/ca.key similarity index 100% rename from pkg/core/proxy/tls/asset/ca.key rename to pkg/agent/proxy/tls/asset/ca.key diff --git a/pkg/core/proxy/tls/asset/setup_ca.sh b/pkg/agent/proxy/tls/asset/setup_ca.sh similarity index 100% rename from pkg/core/proxy/tls/asset/setup_ca.sh rename to pkg/agent/proxy/tls/asset/setup_ca.sh diff --git a/pkg/core/proxy/tls/ca.go b/pkg/agent/proxy/tls/ca.go similarity index 99% rename from pkg/core/proxy/tls/ca.go rename to pkg/agent/proxy/tls/ca.go index 0bd4e68e5a..d0189d1222 100644 --- a/pkg/core/proxy/tls/ca.go +++ b/pkg/agent/proxy/tls/ca.go @@ -18,7 +18,7 @@ import ( cfsslLog "github.com/cloudflare/cfssl/log" "github.com/cloudflare/cfssl/signer" "github.com/cloudflare/cfssl/signer/local" - "go.keploy.io/server/v2/pkg/core/proxy/util" + "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) diff --git a/pkg/core/proxy/tls/tls.go b/pkg/agent/proxy/tls/tls.go similarity index 100% rename from pkg/core/proxy/tls/tls.go rename to pkg/agent/proxy/tls/tls.go diff --git a/pkg/core/proxy/treedb.go b/pkg/agent/proxy/treedb.go similarity index 100% rename from pkg/core/proxy/treedb.go rename to pkg/agent/proxy/treedb.go diff --git a/pkg/core/proxy/util.go b/pkg/agent/proxy/util.go similarity index 97% rename from pkg/core/proxy/util.go rename to pkg/agent/proxy/util.go index 2545148482..5db09b76ef 100644 --- a/pkg/core/proxy/util.go +++ b/pkg/agent/proxy/util.go @@ -9,7 +9,7 @@ import ( "net" "os" - pUtil "go.keploy.io/server/v2/pkg/core/proxy/util" + pUtil "go.keploy.io/server/v2/pkg/agent/proxy/util" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" diff --git a/pkg/core/proxy/util/util.go b/pkg/agent/proxy/util/util.go similarity index 100% rename from pkg/core/proxy/util/util.go rename to pkg/agent/proxy/util/util.go diff --git a/pkg/agent/routes/record.go b/pkg/agent/routes/record.go new file mode 100644 index 0000000000..792619703f --- /dev/null +++ b/pkg/agent/routes/record.go @@ -0,0 +1,215 @@ +// Package routes defines the routes for the agent service. +package routes + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" + "go.keploy.io/server/v2/pkg/service/agent" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type AgentRequest struct { + logger *zap.Logger + agent agent.Service +} + +func New(r chi.Router, agent agent.Service, logger *zap.Logger) { + a := &AgentRequest{ + logger: logger, + agent: agent, + } + + r.Route("/agent", func(r chi.Router) { + r.Get("/health", a.Health) + r.Post("/incoming", a.HandleIncoming) + r.Post("/outgoing", a.HandleOutgoing) + r.Post("/mock", a.MockOutgoing) + r.Post("/setmocks", a.SetMocks) + r.Post("/testbench", a.SendKtInfo) + r.Post("/register", a.RegisterClients) + r.Get("/consumedmocks", a.GetConsumedMocks) + r.Post("/unregister", a.DeRegisterClients) + }) +} + +func (a *AgentRequest) Health(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + render.JSON(w, r, "OK") +} + +func (a *AgentRequest) HandleIncoming(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("Cache-Control", "no-cache") + + // Flush headers to ensure the client gets the response immediately + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Create a context with the request's context to manage cancellation + errGrp, _ := errgroup.WithContext(r.Context()) + ctx := context.WithValue(r.Context(), models.ErrGroupKey, errGrp) + + // decode request body + var incomingReq models.IncomingReq + err := json.NewDecoder(r.Body).Decode(&incomingReq) + if err != nil { + http.Error(w, "Error decoding request", http.StatusBadRequest) + return + } + + // Call GetIncoming to get the channel + tc, err := a.agent.GetIncoming(ctx, incomingReq.ClientID, incomingReq.IncomingOptions) + if err != nil { + http.Error(w, "Error retrieving test cases", http.StatusInternalServerError) + return + } + + // Keep the connection alive and stream data + for t := range tc { + select { + case <-r.Context().Done(): + // Client closed the connection or context was cancelled + return + default: + // Stream each test case as JSON + fmt.Printf("Sending Test case: %v\n", t) + render.JSON(w, r, t) + flusher.Flush() // Immediately send data to the client + } + } +} + +func (a *AgentRequest) HandleOutgoing(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("Cache-Control", "no-cache") + + // Flush headers to ensure the client gets the response immediately + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Create a context with the request's context to manage cancellation + errGrp, _ := errgroup.WithContext(r.Context()) + ctx := context.WithValue(r.Context(), models.ErrGroupKey, errGrp) + + var outgoingReq models.OutgoingReq + err := json.NewDecoder(r.Body).Decode(&outgoingReq) + if err != nil { + http.Error(w, "Error decoding request", http.StatusBadRequest) + return + } + + // Call GetOutgoing to get the channel + mockChan, err := a.agent.GetOutgoing(ctx, outgoingReq.ClientID, outgoingReq.OutgoingOptions) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + for m := range mockChan { + select { + case <-r.Context().Done(): + fmt.Println("Context done in HandleOutgoing") + if m != nil { + render.JSON(w, r, m) + flusher.Flush() + } else { + render.JSON(w, r, "No more mocks") + flusher.Flush() + } + return + default: + // Stream each mock as JSON + render.JSON(w, r, m) + flusher.Flush() + } + } +} + +func (a *AgentRequest) RegisterClients(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var registerReq models.RegisterReq + err := json.NewDecoder(r.Body).Decode(®isterReq) + + register := models.AgentResp{ + ClientID: registerReq.SetupOptions.ClientID, + Error: nil, + } + + if err != nil { + register.Error = err + render.JSON(w, r, register) + render.Status(r, http.StatusBadRequest) + return + } + + fmt.Printf("SetupRequest: %v\n", registerReq.SetupOptions.ClientNsPid) + + if registerReq.SetupOptions.ClientNsPid == 0 { + register.Error = fmt.Errorf("Client pid is required") + render.JSON(w, r, register) + render.Status(r, http.StatusBadRequest) + return + } + fmt.Printf("Register Client req: %v\n", registerReq.SetupOptions) + + err = a.agent.RegisterClient(r.Context(), registerReq.SetupOptions) + if err != nil { + register.Error = err + render.JSON(w, r, register) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, register) + render.Status(r, http.StatusOK) +} + +func (a *AgentRequest) DeRegisterClients(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var UnregisterReq models.UnregisterReq + err := json.NewDecoder(r.Body).Decode(&UnregisterReq) + + mockRes := models.AgentResp{ + ClientID: UnregisterReq.ClientID, + Error: nil, + IsSuccess: true, + } + + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, mockRes) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.DeRegisterClient(r.Context(), UnregisterReq) + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, "Client De-registered") +} diff --git a/pkg/agent/routes/replay.go b/pkg/agent/routes/replay.go new file mode 100644 index 0000000000..a06776577b --- /dev/null +++ b/pkg/agent/routes/replay.go @@ -0,0 +1,101 @@ +// Package routes defines the routes for the agent to mock outgoing requests, set mocks and get consumed mocks. +package routes + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" +) + +func (a *AgentRequest) MockOutgoing(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var OutgoingReq models.OutgoingReq + err := json.NewDecoder(r.Body).Decode(&OutgoingReq) + + mockRes := models.AgentResp{ + ClientID: OutgoingReq.ClientID, + Error: nil, + IsSuccess: true, + } + + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, mockRes) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.MockOutgoing(r.Context(), OutgoingReq.ClientID, OutgoingReq.OutgoingOptions) + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, mockRes) + render.Status(r, http.StatusOK) +} + +func (a *AgentRequest) SetMocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var SetMocksReq models.SetMocksReq + err := json.NewDecoder(r.Body).Decode(&SetMocksReq) + + setmockRes := models.AgentResp{ + ClientID: SetMocksReq.ClientID, + Error: nil, + } + if err != nil { + setmockRes.Error = err + setmockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.SetMocks(r.Context(), SetMocksReq.ClientID, SetMocksReq.Filtered, SetMocksReq.UnFiltered) + if err != nil { + setmockRes.Error = err + setmockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, setmockRes) + render.Status(r, http.StatusOK) + +} + +func (a *AgentRequest) GetConsumedMocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + clientID := r.URL.Query().Get("id") + + // convert string to uint64 + clientIDInt, err := strconv.ParseUint(clientID, 10, 64) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusBadRequest) + return + } + + consumedMocks, err := a.agent.GetConsumedMocks(r.Context(), clientIDInt) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, consumedMocks) + render.Status(r, http.StatusOK) + +} diff --git a/pkg/agent/routes/testbench.go b/pkg/agent/routes/testbench.go new file mode 100644 index 0000000000..1a923d1ca3 --- /dev/null +++ b/pkg/agent/routes/testbench.go @@ -0,0 +1,35 @@ +package routes + +import ( + "encoding/json" + "net/http" + + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" + "go.uber.org/zap" +) + +func (a *AgentRequest) SendKtInfo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var testbenchReq models.TestBenchReq + err := json.NewDecoder(r.Body).Decode(&testbenchReq) + if err != nil { + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.SendKtInfo(r.Context(), testbenchReq) + if err != nil { + a.logger.Error("failed to send kt info", zap.Error(err)) + render.Status(r, http.StatusInternalServerError) + return + } + + tbRes := models.TestBenchResp{ + IsSuccess: true, + Error: "", + } + + render.JSON(w, r, tbRes) + render.Status(r, http.StatusOK) +} diff --git a/pkg/core/service.go b/pkg/agent/service.go similarity index 79% rename from pkg/core/service.go rename to pkg/agent/service.go index 27c43d835f..db2262b4c9 100644 --- a/pkg/core/service.go +++ b/pkg/agent/service.go @@ -1,29 +1,34 @@ //go:build linux -package core +package agent import ( "context" + "fmt" "sync" "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/core/app" - "go.keploy.io/server/v2/pkg/core/hooks/structs" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" + "go.keploy.io/server/v2/pkg/client/app" "go.keploy.io/server/v2/utils" "go.keploy.io/server/v2/pkg/models" ) type Hooks interface { - AppInfo DestInfo OutgoingInfo Load(ctx context.Context, id uint64, cfg HookCfg) error Record(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) + // send KeployClient Pid + SendKeployClientInfo(clientID uint64, clientInfo structs.ClientInfo) error + DeleteKeployClientInfo(clientID uint64) error + SendClientProxyInfo(clientID uint64, proxyInfo structs.ProxyInfo) error + SendKtInfo(ctx context.Context, tb structs.TestBenchInfo) error } type HookCfg struct { - AppID uint64 + ClientID uint64 Pid uint32 IsDocker bool KeployIPV4 string @@ -33,7 +38,7 @@ type HookCfg struct { type App interface { Setup(ctx context.Context, opts app.Options) error - Run(ctx context.Context, inodeChan chan uint64, opts app.Options) error + Run(ctx context.Context, opts app.Options) error Kind(ctx context.Context) utils.CmdType KeployIPv4Addr() string } @@ -45,9 +50,11 @@ type Proxy interface { Mock(ctx context.Context, id uint64, opts models.OutgoingOptions) error SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) + MakeClientDeRegisterd(ctx context.Context) error } type ProxyOptions struct { + ProxyPort uint32 // DNSIPv4Addr is the proxy IP returned by the DNS server. default is loopback address DNSIPv4Addr string // DNSIPv6Addr is the proxy IP returned by the DNS server. default is loopback address @@ -59,18 +66,14 @@ type DestInfo interface { Delete(ctx context.Context, srcPort uint16) error } -type AppInfo interface { - SendDockerAppInfo(id uint64, dockerAppInfo structs.DockerAppInfo) error -} - // For keploy test bench type Tester interface { Setup(ctx context.Context, opts models.TestingOptions) error } type TestBenchInfo interface { - // SendKeployPids(key models.ModeKey, pid uint32) error - // SendKeployPorts(key models.ModeKey, port uint32) error + SendKeployPids(key models.ModeKey, tb structs.TestBenchInfo) error + SendKeployPorts(key models.ModeKey, port uint32) error } // ---------------------- @@ -79,7 +82,7 @@ type OutgoingInfo interface { } type NetworkAddress struct { - AppID uint64 + ClientID uint64 Version uint32 IPv4Addr uint32 IPv6Addr [4]uint32 @@ -97,6 +100,7 @@ func NewSessions() *Sessions { } func (s *Sessions) Get(id uint64) (*Session, bool) { + fmt.Println("Inside Get of Sessions !!", id) v, ok := s.sessions.Load(id) if !ok { return nil, false diff --git a/pkg/core/tester/tester.go b/pkg/agent/tester/tester.go similarity index 79% rename from pkg/core/tester/tester.go rename to pkg/agent/tester/tester.go index 68fbeb0169..aac117f79c 100644 --- a/pkg/core/tester/tester.go +++ b/pkg/agent/tester/tester.go @@ -9,7 +9,8 @@ import ( "fmt" "time" - "go.keploy.io/server/v2/pkg/core" + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/utils" "go.uber.org/zap" @@ -17,10 +18,10 @@ import ( type Tester struct { logger *zap.Logger - testBenchInfo core.TestBenchInfo + testBenchInfo agent.TestBenchInfo } -func New(logger *zap.Logger, testBenchInfo core.TestBenchInfo) *Tester { +func New(logger *zap.Logger, testBenchInfo agent.TestBenchInfo) *Tester { return &Tester{ logger: logger, testBenchInfo: testBenchInfo, @@ -28,8 +29,8 @@ func New(logger *zap.Logger, testBenchInfo core.TestBenchInfo) *Tester { } const ( - testPort = 56789 - recordPort = 36789 + testPort = 46789 + recordPort = 16789 ) func (t *Tester) Setup(ctx context.Context, opts models.TestingOptions) error { @@ -51,6 +52,7 @@ func (t *Tester) Setup(ctx context.Context, opts models.TestingOptions) error { } func (t *Tester) setupReplay(ctx context.Context) error { + fmt.Println("Inside setupReplay") setUpErr := errors.New("failed to setup the keploy replay testing") recordPid, err := utils.GetPIDFromPort(ctx, t.logger, recordPort) @@ -60,13 +62,16 @@ func (t *Tester) setupReplay(ctx context.Context) error { return setUpErr } - t.logger.Debug(fmt.Sprintf("keployRecord pid:%v", recordPid)) + t.logger.Info(fmt.Sprintf("keployRecord pid:%v", recordPid)) - // err = t.testBenchInfo.SendKeployPids(models.RecordKey, recordPid) - // if err != nil { - // utils.LogError(t.logger, err, fmt.Sprintf("failed to send keploy %v server pid to the epbf program", models.MODE_RECORD), zap.Any("Keploy Pid", recordPid)) - // return setUpErr - // } + err = t.testBenchInfo.SendKeployPids(models.RecordKey, structs.TestBenchInfo{ + KRecordAgentPID: recordPid, + }) + + if err != nil { + utils.LogError(t.logger, err, fmt.Sprintf("failed to send keploy %v server pid to the epbf program", models.MODE_RECORD), zap.Any("Keploy Pid", recordPid)) + return setUpErr + } // err = t.testBenchInfo.SendKeployPorts(models.RecordKey, recordPort) // if err != nil { @@ -89,6 +94,7 @@ func (t *Tester) setupReplay(ctx context.Context) error { func (t *Tester) setupRecord(ctx context.Context) error { + fmt.Println("Inside setupRecord") go func() { defer utils.Recover(t.logger) @@ -113,7 +119,7 @@ func (t *Tester) setupRecord(ctx context.Context) error { t.logger.Debug("keploytest pid", zap.Uint32("pid", testPid)) - // // sending keploytest binary pid in keployrecord binary to filter out ingress/egress calls related to keploytest binary. + // sending keploytest binary pid in keployrecord binary to filter out ingress/egress calls related to keploytest binary. // err = t.testBenchInfo.SendKeployPids(models.TestKey, testPid) // if err != nil { // utils.LogError(t.logger, err, fmt.Sprintf("failed to send keploy %v server pid to the epbf program", models.MODE_TEST), zap.Any("Keploy Pid", testPid)) diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go new file mode 100644 index 0000000000..b395d7442a --- /dev/null +++ b/pkg/agent/utils.go @@ -0,0 +1,2 @@ +// Package agent contains backend for the keploy agent. +package agent diff --git a/pkg/core/app/app.go b/pkg/client/app/app.go similarity index 89% rename from pkg/core/app/app.go rename to pkg/client/app/app.go index 5f40049a04..f730ef61a7 100644 --- a/pkg/core/app/app.go +++ b/pkg/client/app/app.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows // Package app provides functionality for managing applications. package app @@ -8,6 +8,7 @@ import ( "errors" "fmt" "os/exec" + "strings" "syscall" "time" @@ -15,7 +16,6 @@ import ( "go.keploy.io/server/v2/pkg/models" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" "go.keploy.io/server/v2/pkg/platform/docker" @@ -49,10 +49,9 @@ type App struct { container string containerNetwork string containerIPv4 chan string - keployNetwork string + KeployNetwork string keployContainer string keployIPv4 string - inodeChan chan uint64 EnableTesting bool Mode models.Mode } @@ -65,6 +64,7 @@ type Options struct { DockerNetwork string } +// Setup sets up the application for running. func (a *App) Setup(_ context.Context) error { if utils.IsDockerCmd(a.kind) && isDetachMode(a.logger, a.cmd, a.kind) { @@ -78,12 +78,12 @@ func (a *App) Setup(_ context.Context) error { return err } case utils.DockerCompose: + a.cmd = a.cmd + " --force-recreate" err := a.SetupCompose() if err != nil { return err } default: - // setup native binary } return nil } @@ -117,6 +117,13 @@ func (a *App) SetupDocker() error { utils.LogError(a.logger, err, fmt.Sprintf("failed to inject network:%v to the keploy container", a.containerNetwork)) return err } + + // attaching the init container's PID namespace to the app container + err = a.attachInitPid(context.Background()) + if err != nil { + utils.LogError(a.logger, err, "failed to attach init pid") + return err + } return nil } @@ -165,7 +172,7 @@ func (a *App) SetupCompose() error { if info == nil { info, err = a.docker.SetKeployNetwork(compose) if err != nil { - utils.LogError(a.logger, nil, "failed to set default network in the compose file", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to set default network in the compose file", zap.String("network", a.KeployNetwork)) return err } composeChanged = true @@ -180,24 +187,33 @@ func (a *App) SetupCompose() error { composeChanged = true } - a.keployNetwork = info.Name - - ok, err = a.docker.NetworkExists(a.keployNetwork) + a.KeployNetwork = info.Name + ok, err = a.docker.NetworkExists(a.KeployNetwork) if err != nil { - utils.LogError(a.logger, nil, "failed to find default network", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to find default network", zap.String("network", a.KeployNetwork)) return err } //if keploy-network doesn't exist locally then create it if !ok { - err = a.docker.CreateNetwork(a.keployNetwork) + err = a.docker.CreateNetwork(a.KeployNetwork) if err != nil { - utils.LogError(a.logger, nil, "failed to create default network", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to create default network", zap.String("network", a.KeployNetwork)) return err } } + //check if compose file has keploy-init container + // adding keploy init pid to the compose file + err = a.docker.SetInitPid(compose, a.container) + if err != nil { + utils.LogError(a.logger, nil, "failed to set init pid in the compose file") + return err + } + composeChanged = true + if composeChanged { + err = a.docker.WriteComposeFile(compose, newPath) if err != nil { utils.LogError(a.logger, nil, "failed to write the compose file", zap.String("path", newPath)) @@ -208,7 +224,7 @@ func (a *App) SetupCompose() error { } if a.containerNetwork == "" { - a.containerNetwork = a.keployNetwork + a.containerNetwork = a.KeployNetwork } err = a.injectNetwork(a.containerNetwork) if err != nil { @@ -233,7 +249,7 @@ func (a *App) injectNetwork(network string) error { return err } - a.keployNetwork = network + a.KeployNetwork = network //sending new proxy ip to kernel, since dynamically injected new network has different ip for keploy. inspect, err := a.docker.ContainerInspect(context.Background(), a.keployContainer) @@ -248,7 +264,7 @@ func (a *App) injectNetwork(network string) error { //TODO: check the logic for correctness for n, settings := range keployNetworks { if n == network { - a.keployIPv4 = settings.IPAddress + a.keployIPv4 = settings.IPAddress // TODO: keployIPv4 needs to be send to the agent a.logger.Info("Successfully injected network to the keploy container", zap.Any("Keploy container", a.keployContainer), zap.Any("appNetwork", network), zap.String("keploy container ip", a.keployIPv4)) return nil } @@ -261,6 +277,25 @@ func (a *App) injectNetwork(network string) error { return fmt.Errorf("failed to find the network:%v in the keploy container", network) } +// AttachInitPid modifies the existing Docker command to attach the init container's PID namespace +func (a *App) attachInitPid(_ context.Context) error { + if a.cmd == "" { + return fmt.Errorf("no command provided to modify") + } + + pidMode := fmt.Sprintf("--pid=container:%s", "keploy-init") + // Inject the pidMode flag after 'docker run' in the command + parts := strings.SplitN(a.cmd, " ", 3) // Split by first two spaces to isolate "docker run" + if len(parts) < 3 { + return fmt.Errorf("invalid command structure: %s", a.cmd) + } + + // Modify the command to insert the pidMode + a.cmd = fmt.Sprintf("%s %s %s %s", parts[0], parts[1], pidMode, parts[2]) + + return nil +} + func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { if e.Action != "start" { return false, nil @@ -290,8 +325,7 @@ func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { return false, err } - a.inodeChan <- inode - a.logger.Debug("container started and successfully extracted inode", zap.Any("inode", inode)) + a.logger.Info("container started and successfully extracted inode", zap.Any("inode", inode)) if info.NetworkSettings == nil || info.NetworkSettings.Networks == nil { a.logger.Debug("container network settings not available", zap.Any("containerDetails.NetworkSettings", info.NetworkSettings)) return false, nil @@ -309,7 +343,6 @@ func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { func (a *App) getDockerMeta(ctx context.Context) <-chan error { // listen for the docker daemon events defer a.logger.Debug("exiting from goroutine of docker daemon event listener") - errCh := make(chan error, 1) timer := time.NewTimer(time.Duration(a.containerDelay) * time.Second) logTicker := time.NewTicker(1 * time.Second) @@ -323,7 +356,7 @@ func (a *App) getDockerMeta(ctx context.Context) <-chan error { filters.KeyValuePair{Key: "action", Value: "start"}, ) - messages, errCh2 := a.docker.Events(ctx, types.EventsOptions{ + messages, errCh2 := a.docker.Events(ctx, events.ListOptions{ Filters: eventFilter, }) @@ -411,12 +444,12 @@ func (a *App) runDocker(ctx context.Context) models.AppError { } return models.AppError{AppErrorType: models.ErrInternal, Err: err} case <-ctx.Done(): + fmt.Println("ctx.Done called in runDocker") return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: ctx.Err()} } } -func (a *App) Run(ctx context.Context, inodeChan chan uint64) models.AppError { - a.inodeChan = inodeChan +func (a *App) Run(ctx context.Context) models.AppError { if utils.IsDockerCmd(a.kind) { return a.runDocker(ctx) @@ -464,7 +497,7 @@ func (a *App) run(ctx context.Context) models.AppError { cmdCancel := func(cmd *exec.Cmd) func() error { return func() error { if utils.IsDockerCmd(a.kind) { - a.logger.Debug("sending SIGINT to the container", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) + a.logger.Info("sending SIGINT to the container", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) err := utils.SendSignal(a.logger, -cmd.Process.Pid, syscall.SIGINT) return err } @@ -473,6 +506,7 @@ func (a *App) run(ctx context.Context) models.AppError { } var err error + cmdErr := utils.ExecuteCommand(ctx, a.logger, userCmd, cmdCancel, 25*time.Second) if cmdErr.Err != nil { switch cmdErr.Type { diff --git a/pkg/core/app/util.go b/pkg/client/app/util.go similarity index 99% rename from pkg/core/app/util.go rename to pkg/client/app/util.go index 6d374fe159..b72bac6964 100644 --- a/pkg/core/app/util.go +++ b/pkg/client/app/util.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package app diff --git a/pkg/core/core_linux.go b/pkg/core/core_linux.go deleted file mode 100644 index 1db78c78f7..0000000000 --- a/pkg/core/core_linux.go +++ /dev/null @@ -1,272 +0,0 @@ -//go:build linux - -// Package core provides functionality for managing core functionalities in Keploy. -package core - -import ( - "context" - "errors" - "fmt" - "sync" - - "golang.org/x/sync/errgroup" - - "go.keploy.io/server/v2/pkg/core/app" - "go.keploy.io/server/v2/pkg/core/hooks/structs" - "go.keploy.io/server/v2/pkg/models" - "go.keploy.io/server/v2/pkg/platform/docker" - "go.keploy.io/server/v2/utils" - "go.uber.org/zap" -) - -type Core struct { - Proxy // embedding the Proxy interface to transfer the proxy methods to the core object - Hooks // embedding the Hooks interface to transfer the hooks methods to the core object - Tester // embedding the Tester interface to transfer the tester methods to the core object - dockerClient docker.Client //embedding the docker client to transfer the docker client methods to the core object - logger *zap.Logger - id utils.AutoInc - apps sync.Map - proxyStarted bool -} - -func New(logger *zap.Logger, hook Hooks, proxy Proxy, tester Tester, client docker.Client) *Core { - return &Core{ - logger: logger, - Hooks: hook, - Proxy: proxy, - Tester: tester, - dockerClient: client, - } -} - -func (c *Core) Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) { - // create a new app and store it in the map - id := uint64(c.id.Next()) - a := app.NewApp(c.logger, id, cmd, c.dockerClient, app.Options{ - DockerNetwork: opts.DockerNetwork, - Container: opts.Container, - DockerDelay: opts.DockerDelay, - }) - c.apps.Store(id, a) - - err := a.Setup(ctx) - if err != nil { - utils.LogError(c.logger, err, "failed to setup app") - return 0, err - } - return id, nil -} - -func (c *Core) getApp(id uint64) (*app.App, error) { - a, ok := c.apps.Load(id) - if !ok { - return nil, fmt.Errorf("app with id:%v not found", id) - } - - // type assertion on the app - h, ok := a.(*app.App) - if !ok { - return nil, fmt.Errorf("failed to type assert app with id:%v", id) - } - - return h, nil -} - -func (c *Core) Hook(ctx context.Context, id uint64, opts models.HookOptions) error { - hookErr := errors.New("failed to hook into the app") - - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return hookErr - } - - isDocker := false - appKind := a.Kind(ctx) - //check if the app is docker/docker-compose or native - if utils.IsDockerCmd(appKind) { - isDocker = true - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) - if !ok { - return errors.New("failed to get the error group from the context") - } - - // create a new error group for the hooks - hookErrGrp, _ := errgroup.WithContext(ctx) - hookCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the hookCtx to control the lifecycle of the hooks - hookCtx, hookCtxCancel := context.WithCancel(hookCtx) - hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) - - // create a new error group for the proxy - proxyErrGrp, _ := errgroup.WithContext(ctx) - proxyCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the proxyCtx to control the lifecycle of the proxy - proxyCtx, proxyCtxCancel := context.WithCancel(proxyCtx) - proxyCtx = context.WithValue(proxyCtx, models.ErrGroupKey, proxyErrGrp) - - g.Go(func() error { - <-ctx.Done() - - proxyCtxCancel() - err = proxyErrGrp.Wait() - if err != nil { - utils.LogError(c.logger, err, "failed to stop the proxy") - } - - hookCtxCancel() - err := hookErrGrp.Wait() - if err != nil { - utils.LogError(c.logger, err, "failed to unload the hooks") - } - - //deleting in order to free the memory in case of rerecord. otherwise different app id will be created for the same app. - c.apps.Delete(id) - c.id = utils.AutoInc{} - - return nil - }) - - //load hooks - err = c.Hooks.Load(hookCtx, id, HookCfg{ - AppID: id, - Pid: 0, - IsDocker: isDocker, - KeployIPV4: a.KeployIPv4Addr(), - Mode: opts.Mode, - Rules: opts.Rules, - }) - if err != nil { - utils.LogError(c.logger, err, "failed to load hooks") - return hookErr - } - - if c.proxyStarted { - c.logger.Debug("Proxy already started") - // return nil - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - // TODO: Hooks can be loaded multiple times but proxy should be started only once - // if there is another containerized app, then we need to pass new (ip:port) of proxy to the eBPF - // as the network namespace is different for each container and so is the keploy/proxy IP to communicate with the app. - // start proxy - err = c.Proxy.StartProxy(proxyCtx, ProxyOptions{ - DNSIPv4Addr: a.KeployIPv4Addr(), - //DnsIPv6Addr: "" - }) - if err != nil { - utils.LogError(c.logger, err, "failed to start proxy") - return hookErr - } - - c.proxyStarted = true - - // For keploy test bench - if opts.EnableTesting { - - // enable testing in the app - a.EnableTesting = true - a.Mode = opts.Mode - - // Setting up the test bench - err := c.Tester.Setup(ctx, models.TestingOptions{Mode: opts.Mode}) - if err != nil { - utils.LogError(c.logger, err, "error while setting up the test bench environment") - return errors.New("failed to setup the test bench") - } - } - - return nil -} - -func (c *Core) Run(ctx context.Context, id uint64, _ models.RunOptions) models.AppError { - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return models.AppError{AppErrorType: models.ErrInternal, Err: err} - } - - runAppErrGrp, runAppCtx := errgroup.WithContext(ctx) - - inodeErrCh := make(chan error, 1) - appErrCh := make(chan models.AppError, 1) - inodeChan := make(chan uint64, 1) //send inode to the hook - - defer func() { - err := runAppErrGrp.Wait() - defer close(inodeErrCh) - defer close(inodeChan) - if err != nil { - utils.LogError(c.logger, err, "failed to stop the app") - } - }() - - runAppErrGrp.Go(func() error { - defer utils.Recover(c.logger) - if a.Kind(ctx) == utils.Native { - return nil - } - select { - case inode := <-inodeChan: - err := c.Hooks.SendDockerAppInfo(id, structs.DockerAppInfo{AppInode: inode, ClientID: id}) - if err != nil { - utils.LogError(c.logger, err, "") - - inodeErrCh <- errors.New("failed to send inode to the kernel") - } - case <-ctx.Done(): - return nil - } - return nil - }) - - runAppErrGrp.Go(func() error { - defer utils.Recover(c.logger) - defer close(appErrCh) - appErr := a.Run(runAppCtx, inodeChan) - if appErr.Err != nil { - utils.LogError(c.logger, appErr.Err, "error while running the app") - appErrCh <- appErr - } - return nil - }) - - select { - case <-runAppCtx.Done(): - return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: nil} - case appErr := <-appErrCh: - return appErr - case inodeErr := <-inodeErrCh: - return models.AppError{AppErrorType: models.ErrInternal, Err: inodeErr} - } -} - -func (c *Core) GetContainerIP(_ context.Context, id uint64) (string, error) { - - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return "", err - } - - ip := a.ContainerIPv4Addr() - c.logger.Debug("ip address of the target app container", zap.Any("ip", ip)) - if ip == "" { - return "", fmt.Errorf("failed to get the IP address of the app container. Try increasing --delay (in seconds)") - } - - return ip, nil -} diff --git a/pkg/core/hooks/bpf_arm64_bpfel.o b/pkg/core/hooks/bpf_arm64_bpfel.o deleted file mode 100644 index f8c9fe223f..0000000000 Binary files a/pkg/core/hooks/bpf_arm64_bpfel.o and /dev/null differ diff --git a/pkg/core/hooks/bpf_x86_bpfel.o b/pkg/core/hooks/bpf_x86_bpfel.o deleted file mode 100644 index f3f76af725..0000000000 Binary files a/pkg/core/hooks/bpf_x86_bpfel.o and /dev/null differ diff --git a/pkg/core/hooks/kernelComm.go b/pkg/core/hooks/kernelComm.go deleted file mode 100644 index abf8d98d07..0000000000 --- a/pkg/core/hooks/kernelComm.go +++ /dev/null @@ -1,104 +0,0 @@ -//go:build linux - -package hooks - -import ( - "context" - "fmt" - - "math/rand" - - "github.com/cilium/ebpf" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/hooks/structs" - "go.keploy.io/server/v2/utils" - "go.uber.org/zap" -) - -//TODO: rename this file. - -// Get Used by proxy -func (h *Hooks) Get(_ context.Context, srcPort uint16) (*core.NetworkAddress, error) { - d, err := h.GetDestinationInfo(srcPort) - if err != nil { - return nil, err - } - // TODO : need to implement eBPF code to differentiate between different apps - s, ok := h.sess.Get(0) - if !ok { - return nil, fmt.Errorf("session not found") - } - - return &core.NetworkAddress{ - AppID: s.ID, - Version: d.IPVersion, - IPv4Addr: d.DestIP4, - IPv6Addr: d.DestIP6, - Port: d.DestPort, - }, nil -} - -// GetDestinationInfo retrieves destination information associated with a source port. -func (h *Hooks) GetDestinationInfo(srcPort uint16) (*structs.DestInfo, error) { - h.m.Lock() - defer h.m.Unlock() - destInfo := structs.DestInfo{} - if err := h.redirectProxyMap.Lookup(srcPort, &destInfo); err != nil { - return nil, err - } - return &destInfo, nil -} - -func (h *Hooks) Delete(_ context.Context, srcPort uint16) error { - return h.CleanProxyEntry(srcPort) -} - -func (h *Hooks) CleanProxyEntry(srcPort uint16) error { - h.m.Lock() - defer h.m.Unlock() - err := h.redirectProxyMap.Delete(srcPort) - if err != nil { - utils.LogError(h.logger, err, "failed to remove entry from redirect proxy map") - return err - } - h.logger.Debug("successfully removed entry from redirect proxy map", zap.Any("(Key)/SourcePort", srcPort)) - return nil -} - -func (h *Hooks) SendClientInfo(id uint64, appInfo structs.ClientInfo) error { - err := h.clientRegistrationMap.Update(id, appInfo, ebpf.UpdateAny) - if err != nil { - utils.LogError(h.logger, err, "failed to send the app info to the ebpf program") - return err - } - return nil -} - -func (h *Hooks) SendAgentInfo(agentInfo structs.AgentInfo) error { - key := 0 - err := h.agentRegistartionMap.Update(uint32(key), agentInfo, ebpf.UpdateAny) - if err != nil { - utils.LogError(h.logger, err, "failed to send the agent info to the ebpf program") - return err - } - return nil -} - -func (h *Hooks) SendDockerAppInfo(_ uint64, dockerAppInfo structs.DockerAppInfo) error { - if h.appID != 0 { - err := h.dockerAppRegistrationMap.Delete(h.appID) - if err != nil { - utils.LogError(h.logger, err, "failed to remove entry from dockerAppRegistrationMap") - return err - } - } - r := rand.New(rand.NewSource(rand.Int63())) - randomNum := r.Uint64() - h.appID = randomNum - err := h.dockerAppRegistrationMap.Update(h.appID, dockerAppInfo, ebpf.UpdateAny) - if err != nil { - utils.LogError(h.logger, err, "failed to send the dockerAppInfo info to the ebpf program") - return err - } - return nil -} diff --git a/pkg/core/proxy/parsers.go b/pkg/core/proxy/parsers.go deleted file mode 100644 index aacc44e513..0000000000 --- a/pkg/core/proxy/parsers.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build linux - -package proxy - -import ( - // import all the integrations - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/generic" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/grpc" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/http" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/mongo" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/postgres/v1" - _ "go.keploy.io/server/v2/pkg/core/proxy/integrations/redis" -) diff --git a/pkg/core/record.go b/pkg/core/record.go deleted file mode 100644 index 471ebf40e5..0000000000 --- a/pkg/core/record.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build linux - -package core - -import ( - "context" - - "go.keploy.io/server/v2/pkg/models" -) - -func (c *Core) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - return c.Hooks.Record(ctx, id, opts) -} - -func (c *Core) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { - m := make(chan *models.Mock, 500) - - err := c.Proxy.Record(ctx, id, m, opts) - if err != nil { - return nil, err - } - - return m, nil -} diff --git a/pkg/core/replay.go b/pkg/core/replay.go deleted file mode 100644 index 4bc6cfdd6e..0000000000 --- a/pkg/core/replay.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build linux - -package core - -import ( - "context" - - "go.keploy.io/server/v2/pkg/models" -) - -func (c *Core) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { - - err := c.Proxy.Mock(ctx, id, opts) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/models/agent.go b/pkg/models/agent.go new file mode 100644 index 0000000000..6c3df53584 --- /dev/null +++ b/pkg/models/agent.go @@ -0,0 +1,36 @@ +package models + +type OutgoingReq struct { + OutgoingOptions OutgoingOptions `json:"outgoingOptions"` + ClientID uint64 `json:"clientId"` +} + +type IncomingReq struct { + IncomingOptions IncomingOptions `json:"incomingOptions"` + ClientID uint64 `json:"clientId"` +} + +type RegisterReq struct { + SetupOptions SetupOptions `json:"setupOptions"` +} + +type AgentResp struct { + ClientID uint64 `json:"clientID"` // uuid of the app + Error error `json:"error"` + IsSuccess bool `json:"isSuccess"` +} + +type RunReq struct { + RunOptions RunOptions `json:"runOptions"` + ClientID uint64 `json:"clientId"` +} + +type SetMocksReq struct { + Filtered []*Mock `json:"filtered"` + UnFiltered []*Mock `json:"unFiltered"` + ClientID uint64 `json:"clientId"` +} +type UnregisterReq struct { + ClientID uint64 `json:"clientId"` + Mode Mode `json:"mode"` +} diff --git a/pkg/models/instrument.go b/pkg/models/instrument.go index 15a6dad31b..ea4c448b91 100644 --- a/pkg/models/instrument.go +++ b/pkg/models/instrument.go @@ -11,16 +11,19 @@ type HookOptions struct { Rules []config.BypassRule Mode Mode EnableTesting bool + IsDocker bool + ProxyPort uint32 + ServerPort uint32 } type OutgoingOptions struct { Rules []config.BypassRule MongoPassword string // TODO: role of SQLDelay should be mentioned in the comments. - SQLDelay time.Duration // This is the same as Application delay. - FallBackOnMiss bool // this enables to pass the request to the actual server if no mock is found during test mode. - Mocking bool // used to enable/disable mocking - DstCfg *ConditionalDstCfg + SQLDelay time.Duration // This is the same as Application delay. + FallBackOnMiss bool // this enables to pass the request to the actual server if no mock is found during test mode. + Mocking bool // used to enable/disable mocking + DstCfg *ConditionalDstCfg `json:"-"` } type ConditionalDstCfg struct { @@ -34,9 +37,19 @@ type IncomingOptions struct { } type SetupOptions struct { + ClientID uint64 Container string DockerNetwork string DockerDelay uint64 + ClientNsPid uint32 + ClientInode uint64 + AppInode uint64 + Cmd string + IsDocker bool + CommandType string + EnableTesting bool + ProxyPort uint32 + Mode Mode } type RunOptions struct { diff --git a/pkg/models/mock.go b/pkg/models/mock.go index d0d0838df6..42e0456abd 100755 --- a/pkg/models/mock.go +++ b/pkg/models/mock.go @@ -6,6 +6,20 @@ import ( "go.keploy.io/server/v2/pkg/models/mysql" ) +type MockResponse struct { + Type ResponseType `json:"Type,omitempty" bson:"Type,omitempty"` + Error error + Mock *Mock +} + +type ResponseType string + +const ( + EOF ResponseType = "EOF" + PACKET ResponseType = "PACKET" + ERROR ResponseType = "ERROR" +) + type Mock struct { Version Version `json:"Version,omitempty" bson:"Version,omitempty"` Name string `json:"Name,omitempty" bson:"Name,omitempty"` diff --git a/pkg/models/mysql/comm.go b/pkg/models/mysql/comm.go index c0ed297ca8..2dfef943f4 100644 --- a/pkg/models/mysql/comm.go +++ b/pkg/models/mysql/comm.go @@ -1,4 +1,4 @@ -// Package mysql in models provides realted structs for mysql protocol +// Package mysql in models provides related structs for mysql protocol package mysql // This file contains struct for command phase packets @@ -13,157 +13,157 @@ package mysql // COM_QUERY packet (currently does not support if CLIENT_QUERY_ATTRIBUTES is set) type QueryPacket struct { - Command byte `yaml:"command"` - Query string `yaml:"query"` + Command byte `yaml:"command" json:"command"` + Query string `yaml:"query" json:"query"` } // LocalInFileRequestPacket is used to send local file request to server, currently not supported type LocalInFileRequestPacket struct { - PacketType byte `yaml:"command"` - Filename string + PacketType byte `yaml:"command" json:"command"` + Filename string `yaml:"filename" json:"filename"` } // TextResultSet is used as a response packet for COM_QUERY type TextResultSet struct { - ColumnCount uint64 `yaml:"columnCount"` - Columns []*ColumnDefinition41 `yaml:"columns"` - EOFAfterColumns []byte `yaml:"eofAfterColumns"` - Rows []*TextRow `yaml:"rows"` - FinalResponse *GenericResponse `yaml:"FinalResponse"` + ColumnCount uint64 `yaml:"columnCount" json:"columnCount"` + Columns []*ColumnDefinition41 `yaml:"columns" json:"columns"` + EOFAfterColumns []byte `yaml:"eofAfterColumns" json:"eofAfterColumns"` + Rows []*TextRow `yaml:"rows" json:"rows"` + FinalResponse *GenericResponse `yaml:"FinalResponse" json:"FinalResponse"` } // BinaryProtocolResultSet is used as a response packet for COM_STMT_EXECUTE type BinaryProtocolResultSet struct { - ColumnCount uint64 `yaml:"columnCount"` - Columns []*ColumnDefinition41 `yaml:"columns"` - EOFAfterColumns []byte `yaml:"eofAfterColumns"` - Rows []*BinaryRow `yaml:"rows"` - FinalResponse *GenericResponse `yaml:"FinalResponse"` + ColumnCount uint64 `yaml:"columnCount" json:"columnCount"` + Columns []*ColumnDefinition41 `yaml:"columns" json:"columns"` + EOFAfterColumns []byte `yaml:"eofAfterColumns" json:"eofAfterColumns"` + Rows []*BinaryRow `yaml:"rows" json:"rows"` + FinalResponse *GenericResponse `yaml:"FinalResponse" json:"FinalResponse"` } type GenericResponse struct { - Data []byte `yaml:"data"` - Type string `yaml:"type"` + Data []byte `yaml:"data" json:"data"` + Type string `yaml:"type" json:"type"` } // Columns type ColumnCount struct { - // Header Header `yaml:"header"` - Count uint64 `yaml:"count"` + // Header Header `yaml:"header" json:"header"` + Count uint64 `yaml:"count" json:"count"` } type ColumnDefinition41 struct { - Header Header `yaml:"header"` - Catalog string `yaml:"catalog"` - Schema string `yaml:"schema"` - Table string `yaml:"table"` - OrgTable string `yaml:"org_table"` - Name string `yaml:"name"` - OrgName string `yaml:"org_name"` - FixedLength byte `yaml:"fixed_length"` - CharacterSet uint16 `yaml:"character_set"` - ColumnLength uint32 `yaml:"column_length"` - Type byte `yaml:"type"` - Flags uint16 `yaml:"flags"` - Decimals byte `yaml:"decimals"` - Filler []byte `yaml:"filler"` - DefaultValue string `yaml:"defaultValue"` -} - -//Rows + Header Header `yaml:"header" json:"header"` + Catalog string `yaml:"catalog" json:"catalog"` + Schema string `yaml:"schema" json:"schema"` + Table string `yaml:"table" json:"table"` + OrgTable string `yaml:"org_table" json:"org_table"` + Name string `yaml:"name" json:"name"` + OrgName string `yaml:"org_name" json:"org_name"` + FixedLength byte `yaml:"fixed_length" json:"fixed_length"` + CharacterSet uint16 `yaml:"character_set" json:"character_set"` + ColumnLength uint32 `yaml:"column_length" json:"column_length"` + Type byte `yaml:"type" json:"type"` + Flags uint16 `yaml:"flags" json:"flags"` + Decimals byte `yaml:"decimals" json:"decimals"` + Filler []byte `yaml:"filler" json:"filler"` + DefaultValue string `yaml:"defaultValue" json:"defaultValue"` +} + +// Rows type TextRow struct { - Header Header `yaml:"header"` - Values []ColumnEntry `yaml:"values"` + Header Header `yaml:"header" json:"header"` + Values []ColumnEntry `yaml:"values" json:"values"` } type BinaryRow struct { - Header Header `yaml:"header"` - Values []ColumnEntry `yaml:"values"` - OkAfterRow bool `yaml:"okAfterRow"` - RowNullBuffer []byte `yaml:"rowNullBuffer"` + Header Header `yaml:"header" json:"header"` + Values []ColumnEntry `yaml:"values" json:"values"` + OkAfterRow bool `yaml:"okAfterRow" json:"okAfterRow"` + RowNullBuffer []byte `yaml:"rowNullBuffer" json:"rowNullBuffer"` } type ColumnEntry struct { - Type FieldType `yaml:"type"` - Name string `yaml:"name"` - Value interface{} `yaml:"value"` - Unsigned bool `yaml:"unsigned"` + Type FieldType `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` + Value interface{} `yaml:"value" json:"value"` + Unsigned bool `yaml:"unsigned" json:"unsigned"` } // COM_STMT_PREPARE packet type StmtPreparePacket struct { - Command byte `yaml:"command"` - Query string `yaml:"query"` + Command byte `yaml:"command" json:"command"` + Query string `yaml:"query" json:"query"` } // COM_STMT_PREPARE_OK packet type StmtPrepareOkPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - NumColumns uint16 `yaml:"num_columns"` - NumParams uint16 `yaml:"num_params"` - Filler byte `yaml:"filler"` - WarningCount uint16 `yaml:"warning_count"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + NumColumns uint16 `yaml:"num_columns" json:"num_columns"` + NumParams uint16 `yaml:"num_params" json:"num_params"` + Filler byte `yaml:"filler" json:"filler"` + WarningCount uint16 `yaml:"warning_count" json:"warning_count"` - ParamDefs []*ColumnDefinition41 `yaml:"param_definitions"` - EOFAfterParamDefs []byte `yaml:"eofAfterParamDefs"` - ColumnDefs []*ColumnDefinition41 `yaml:"column_definitions"` - EOFAfterColumnDefs []byte `yaml:"eofAfterColumnDefs"` + ParamDefs []*ColumnDefinition41 `yaml:"param_definitions" json:"param_definitions"` + EOFAfterParamDefs []byte `yaml:"eofAfterParamDefs" json:"eofAfterParamDefs"` + ColumnDefs []*ColumnDefinition41 `yaml:"column_definitions" json:"column_definitions"` + EOFAfterColumnDefs []byte `yaml:"eofAfterColumnDefs" json:"eofAfterColumnDefs"` } // COM_STMT_EXECUTE packet type StmtExecutePacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - Flags byte `yaml:"flags"` - IterationCount uint32 `yaml:"iteration_count"` - ParameterCount int `yaml:"parameter_count"` - NullBitmap []byte `yaml:"null_bitmap"` - NewParamsBindFlag byte `yaml:"new_params_bind_flag"` - Parameters []Parameter `yaml:"parameters"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + Flags byte `yaml:"flags" json:"flags"` + IterationCount uint32 `yaml:"iteration_count" json:"iteration_count"` + ParameterCount int `yaml:"parameter_count" json:"parameter_count"` + NullBitmap []byte `yaml:"null_bitmap" json:"null_bitmap"` + NewParamsBindFlag byte `yaml:"new_params_bind_flag" json:"new_params_bind_flag"` + Parameters []Parameter `yaml:"parameters" json:"parameters"` } type Parameter struct { - Type uint16 `yaml:"type"` - Unsigned bool `yaml:"unsigned"` - Name string `yaml:"name,omitempty"` - Value []byte `yaml:"value"` + Type uint16 `yaml:"type" json:"type"` + Unsigned bool `yaml:"unsigned" json:"unsigned"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Value []byte `yaml:"value" json:"value"` } // COM_STMT_FETCH packet is not currently supported because its response involves multi-resultset type StmtFetchPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - NumRows uint32 `yaml:"num_rows"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + NumRows uint32 `yaml:"num_rows" json:"num_rows"` } // COM_STMT_CLOSE packet type StmtClosePacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` } // COM_STMT_RESET packet type StmtResetPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` } // COM_STMT_SEND_LONG_DATA packet type StmtSendLongDataPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - ParameterID uint16 `yaml:"parameter_id"` - Data []byte `yaml:"data"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + ParameterID uint16 `yaml:"parameter_id" json:"parameter_id"` + Data []byte `yaml:"data" json:"data"` } // Utility commands @@ -171,51 +171,51 @@ type StmtSendLongDataPacket struct { // COM_QUIT packet type QuitPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_INIT_DB packet type InitDBPacket struct { - Command byte `yaml:"command"` - Schema string `yaml:"schema"` + Command byte `yaml:"command" json:"command"` + Schema string `yaml:"schema" json:"schema"` } // COM_STATISTICS packet type StatisticsPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_DEBUG packet type DebugPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_PING packet type PingPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_RESET_CONNECTION packet type ResetConnectionPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_SET_OPTION packet type SetOptionPacket struct { - Status byte `yaml:"status"` - Option uint16 `yaml:"option"` + Status byte `yaml:"status" json:"status"` + Option uint16 `yaml:"option" json:"option"` } // COM_CHANGE_USER packet (Not completed/supported as of now) //refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_change_user.html type ChangeUserPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` // rest of the fields are not present as the packet is not supported } diff --git a/pkg/models/mysql/conn.go b/pkg/models/mysql/conn.go index 51af3a66dd..5045a70190 100644 --- a/pkg/models/mysql/conn.go +++ b/pkg/models/mysql/conn.go @@ -7,29 +7,29 @@ package mysql // HandshakeV10Packet represents the initial handshake packet sent by the server to the client type HandshakeV10Packet struct { - ProtocolVersion uint8 `yaml:"protocol_version"` - ServerVersion string `yaml:"server_version"` - ConnectionID uint32 `yaml:"connection_id"` - AuthPluginData []byte `yaml:"auth_plugin_data,omitempty,flow"` - Filler byte `yaml:"filler"` - CapabilityFlags uint32 `yaml:"capability_flags"` - CharacterSet uint8 `yaml:"character_set"` - StatusFlags uint16 `yaml:"status_flags"` - AuthPluginName string `yaml:"auth_plugin_name"` + ProtocolVersion uint8 `yaml:"protocol_version" json:"protocol_version"` + ServerVersion string `yaml:"server_version" json:"server_version"` + ConnectionID uint32 `yaml:"connection_id" json:"connection_id"` + AuthPluginData []byte `yaml:"auth_plugin_data,omitempty,flow" json:"auth_plugin_data,omitempty"` + Filler byte `yaml:"filler" json:"filler"` + CapabilityFlags uint32 `yaml:"capability_flags" json:"capability_flags"` + CharacterSet uint8 `yaml:"character_set" json:"character_set"` + StatusFlags uint16 `yaml:"status_flags" json:"status_flags"` + AuthPluginName string `yaml:"auth_plugin_name" json:"auth_plugin_name"` } // HandshakeResponse41Packet represents the response packet sent by the client to the server after receiving the HandshakeV10Packet type HandshakeResponse41Packet struct { - CapabilityFlags uint32 `yaml:"capability_flags"` - MaxPacketSize uint32 `yaml:"max_packet_size"` - CharacterSet uint8 `yaml:"character_set"` - Filler [23]byte `yaml:"filler,omitempty,flow"` - Username string `yaml:"username"` - AuthResponse []byte `yaml:"auth_response,omitempty,flow"` - Database string `yaml:"database"` - AuthPluginName string `yaml:"auth_plugin_name"` - ConnectionAttributes map[string]string `yaml:"connection_attributes,omitempty"` - ZstdCompressionLevel byte `yaml:"zstdcompressionlevel"` + CapabilityFlags uint32 `yaml:"capability_flags" json:"capability_flags"` + MaxPacketSize uint32 `yaml:"max_packet_size" json:"max_packet_size"` + CharacterSet uint8 `yaml:"character_set" json:"character_set"` + Filler [23]byte `yaml:"filler,omitempty,flow" json:"filler,omitempty"` + Username string `yaml:"username" json:"username"` + AuthResponse []byte `yaml:"auth_response,omitempty,flow" json:"auth_response,omitempty"` + Database string `yaml:"database" json:"database"` + AuthPluginName string `yaml:"auth_plugin_name" json:"auth_plugin_name"` + ConnectionAttributes map[string]string `yaml:"connection_attributes,omitempty" json:"connection_attributes,omitempty"` + ZstdCompressionLevel byte `yaml:"zstdcompressionlevel" json:"zstdcompressionlevel"` } type SSLRequestPacket struct { @@ -43,26 +43,26 @@ type SSLRequestPacket struct { // AuthSwitchRequestPacket represents the packet sent by the server to the client to switch to a different authentication method type AuthSwitchRequestPacket struct { - StatusTag byte `yaml:"status_tag"` - PluginName string `yaml:"plugin_name"` - PluginData string `yaml:"plugin_data"` + StatusTag byte `yaml:"status_tag" json:"status_tag"` + PluginName string `yaml:"plugin_name" json:"plugin_name"` + PluginData string `yaml:"plugin_data" json:"plugin_data"` } // AuthSwitchResponsePacket represents the packet sent by the client to the server in response to an AuthSwitchRequestPacket. // Note: If the server sends an AuthMoreDataPacket, the client will continue sending AuthSwitchResponsePackets until the server sends an OK packet or an ERR packet. type AuthSwitchResponsePacket struct { - Data string `yaml:"data"` + Data string `yaml:"data" json:"data"` } // AuthMoreDataPacket represents the packet sent by the server to the client to request additional data for authentication type AuthMoreDataPacket struct { - StatusTag byte `yaml:"status_tag"` - Data string `yaml:"data"` + StatusTag byte `yaml:"status_tag" json:"status_tag"` + Data string `yaml:"data" json:"data"` } // AuthNextFactorPacket represents the packet sent by the server to the client to request the next factor for multi-factor authentication type AuthNextFactorPacket struct { - PacketType byte `yaml:"packet_type"` - PluginName string `yaml:"plugin_name"` - PluginData string `yaml:"plugin_data"` + PacketType byte `yaml:"packet_type" json:"packet_type"` + PluginName string `yaml:"plugin_name" json:"plugin_name"` + PluginData string `yaml:"plugin_data" json:"plugin_data"` } diff --git a/pkg/models/mysql/generic.go b/pkg/models/mysql/generic.go index 07be522951..358dea3b19 100644 --- a/pkg/models/mysql/generic.go +++ b/pkg/models/mysql/generic.go @@ -1,7 +1,7 @@ package mysql // This file contains structs for mysql generic response packets -//refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_response_packets.html +// refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_response_packets.html // OKPacket represents the OK packet sent by the server to the client, it represents a successful completion of a command type OKPacket struct { @@ -15,16 +15,16 @@ type OKPacket struct { // ERRPacket represents the ERR packet sent by the server to the client, it represents an error occurred during the execution of a command type ERRPacket struct { - Header byte `yaml:"header"` - ErrorCode uint16 `yaml:"error_code"` - SQLStateMarker string `yaml:"sql_state_marker"` - SQLState string `yaml:"sql_state"` - ErrorMessage string `yaml:"error_message"` + Header byte `json:"header" yaml:"header"` + ErrorCode uint16 `json:"error_code" yaml:"error_code"` + SQLStateMarker string `json:"sql_state_marker" yaml:"sql_state_marker"` + SQLState string `json:"sql_state" yaml:"sql_state"` + ErrorMessage string `json:"error_message" yaml:"error_message"` } // EOFPacket represents the EOF packet sent by the server to the client, it represents the end of a query execution result type EOFPacket struct { - Header byte `yaml:"header"` - Warnings uint16 `yaml:"warnings"` - StatusFlags uint16 `yaml:"status_flags"` + Header byte `json:"header" yaml:"header"` + Warnings uint16 `json:"warnings" yaml:"warnings"` + StatusFlags uint16 `json:"status_flags" yaml:"status_flags"` } diff --git a/pkg/models/mysql/mysql.go b/pkg/models/mysql/mysql.go index c1f6389a7e..12c790e1ba 100644 --- a/pkg/models/mysql/mysql.go +++ b/pkg/models/mysql/mysql.go @@ -1,6 +1,8 @@ package mysql import ( + "encoding/json" + "errors" "time" "gopkg.in/yaml.v3" @@ -59,3 +61,299 @@ type Header struct { PayloadLength uint32 `json:"payload_length" yaml:"payload_length"` SequenceID uint8 `json:"sequence_id" yaml:"sequence_id"` } + +// custom marshal and unmarshal methods for Request and Response structs + +// MarshalJSON implements json.Marshaler for Request because of interface type of field 'Message' +func (r *Request) MarshalJSON() ([]byte, error) { + // create an alias struct to avoid infinite recursion + type RequestAlias struct { + Header *PacketInfo `json:"header"` + Message json.RawMessage `json:"message"` + Meta map[string]string `json:"meta,omitempty"` + } + + aux := RequestAlias{ + Header: r.Header, + Message: json.RawMessage(nil), + Meta: r.Meta, + } + + if r.Message != nil { + // Marshal the message interface{} into JSON + msgJSON, err := json.Marshal(r.Message) + if err != nil { + return nil, err + } + aux.Message = msgJSON + } + + // Marshal the alias struct into JSON + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler for Request because of interface type of field 'Message' +func (r *Request) UnmarshalJSON(data []byte) error { + // Alias struct to prevent recursion during unmarshalling + type RequestAlias struct { + Header *PacketInfo `json:"header"` + Message json.RawMessage `json:"message"` + Meta map[string]string `json:"meta,omitempty"` + } + var aux RequestAlias + + // Unmarshal the data into the alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Assign the unmarshalled data to the original struct + r.Header = aux.Header + r.Meta = aux.Meta + + // Unmarshal the message field based on the type in the header + switch r.Header.Type { + case HandshakeResponse41: + var msg HandshakeResponse41Packet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CachingSha2PasswordToString(RequestPublicKey): + var msg string + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = msg + + case "encrypted_password": + var msg string + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = msg + + case CommandStatusToString(COM_QUIT): + var msg QuitPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_INIT_DB): + var msg InitDBPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STATISTICS): + var msg StatisticsPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_DEBUG): + var msg DebugPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_PING): + var msg PingPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_CHANGE_USER): + var msg ChangeUserPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_RESET_CONNECTION): + var msg ResetConnectionPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_QUERY): + var msg QueryPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_PREPARE): + var msg StmtPreparePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_EXECUTE): + var msg StmtExecutePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_CLOSE): + var msg StmtClosePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_RESET): + var msg StmtResetPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_SEND_LONG_DATA): + var msg StmtSendLongDataPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + default: + return errors.New("failed to unmarshal unknown request packet type") + } + return nil +} + +// MarshalJSON implements json.Marshaler for Response because of interface type of field 'Message' +func (r *Response) MarshalJSON() ([]byte, error) { + // Alias to avoid recursion + type ResponseAlias struct { + PacketBundle `json:"packet_bundle"` + Payload string `json:"payload,omitempty"` + Message json.RawMessage `json:"message"` + } + + aux := ResponseAlias{ + PacketBundle: r.PacketBundle, + Payload: r.Payload, + } + + if r.Message != nil { + // Marshal the message interface{} into JSON + msgJSON, err := json.Marshal(r.Message) + if err != nil { + return nil, err + } + aux.Message = msgJSON + } + + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler for Response because of interface type of field 'Message' +func (r *Response) UnmarshalJSON(data []byte) error { + // Alias struct to prevent recursion + type ResponseAlias struct { + PacketBundle `json:"packet_bundle"` + Payload string `json:"payload,omitempty"` + Message json.RawMessage `json:"message"` + } + var aux ResponseAlias + + // Unmarshal the data into the alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Assign the unmarshalled data to the original struct + r.PacketBundle = aux.PacketBundle + r.Payload = aux.Payload + + // Unmarshal the message field based on the type in the header + switch r.PacketBundle.Header.Type { + // Generic response + case StatusToString(EOF): + var msg EOFPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case StatusToString(ERR): + var msg ERRPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case StatusToString(OK): + var msg OKPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + // Connection phase + case AuthStatusToString(HandshakeV10): + var msg HandshakeV10Packet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthSwitchRequest): + var msg AuthSwitchRequestPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthMoreData): + var msg AuthMoreDataPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthNextFactor): // not supported yet + var msg AuthNextFactorPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + // Command phase + case COM_STMT_PREPARE_OK: + var msg StmtPrepareOkPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case string(Text): + var msg TextResultSet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case string(Binary): + var msg BinaryProtocolResultSet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + default: + return errors.New("failed to unmarshal unknown response packet type") + } + + return nil +} diff --git a/pkg/models/postgres.go b/pkg/models/postgres.go index bb8306b076..faa4da6fdc 100755 --- a/pkg/models/postgres.go +++ b/pkg/models/postgres.go @@ -34,12 +34,12 @@ type Backend struct { CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` Describe pgproto3.Describe `json:"describe,omitempty" yaml:"describe,omitempty"` - Execute pgproto3.Execute `yaml:"-"` + Execute pgproto3.Execute `json:"-" yaml:"-"` Executes []pgproto3.Execute `json:"execute,omitempty" yaml:"execute,omitempty"` Flush pgproto3.Flush `json:"flush,omitempty" yaml:"flush,omitempty"` FunctionCall pgproto3.FunctionCall `json:"function_call,omitempty" yaml:"function_call,omitempty"` GssEncRequest pgproto3.GSSEncRequest `json:"gss_enc_request,omitempty" yaml:"gss_enc_request,omitempty"` - Parse pgproto3.Parse `yaml:"-"` + Parse pgproto3.Parse `json:"-" yaml:"-"` Parses []pgproto3.Parse `json:"parse,omitempty" yaml:"parse,omitempty"` Query pgproto3.Query `json:"query,omitempty" yaml:"query,omitempty"` SSlRequest pgproto3.SSLRequest `json:"ssl_request,omitempty" yaml:"ssl_request,omitempty"` @@ -70,45 +70,33 @@ type Frontend struct { AuthenticationSASLContinue pgproto3.AuthenticationSASLContinue `json:"authentication_sasl_continue,omitempty" yaml:"authentication_sasl_continue,omitempty,flow"` AuthenticationSASLFinal pgproto3.AuthenticationSASLFinal `json:"authentication_sasl_final,omitempty" yaml:"authentication_sasl_final,omitempty,flow"` BackendKeyData pgproto3.BackendKeyData `json:"backend_key_data,omitempty" yaml:"backend_key_data,omitempty"` - BindComplete pgproto3.BindComplete `yaml:"-"` + BindComplete pgproto3.BindComplete `json:"-" yaml:"-"` BindCompletes []pgproto3.BindComplete `json:"bind_complete,omitempty" yaml:"bind_complete,omitempty"` CloseComplete pgproto3.CloseComplete `json:"close_complete,omitempty" yaml:"close_complete,omitempty"` - CommandComplete pgproto3.CommandComplete `yaml:"-"` + CommandComplete pgproto3.CommandComplete `json:"-" yaml:"-"` CommandCompletes []pgproto3.CommandComplete `json:"command_complete,omitempty" yaml:"command_complete,omitempty"` - CopyBothResponse pgproto3.CopyBothResponse `json:"copy_both_response,omitempty" yaml:"copy_both_response,omitempty"` - CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` - CopyInResponse pgproto3.CopyInResponse `json:"copy_in_response,omitempty" yaml:"copy_in_response,omitempty"` - CopyOutResponse pgproto3.CopyOutResponse `json:"copy_out_response,omitempty" yaml:"copy_out_response,omitempty"` - CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` - DataRow pgproto3.DataRow `yaml:"-"` - DataRows []pgproto3.DataRow `json:"data_row,omitempty" yaml:"data_row,omitempty,flow"` - EmptyQueryResponse pgproto3.EmptyQueryResponse `json:"empty_query_response,omitempty" yaml:"empty_query_response,omitempty"` - ErrorResponse pgproto3.ErrorResponse `json:"error_response,omitempty" yaml:"error_response,omitempty"` - FunctionCallResponse pgproto3.FunctionCallResponse `json:"function_call_response,omitempty" yaml:"function_call_response,omitempty"` - NoData pgproto3.NoData `json:"no_data,omitempty" yaml:"no_data,omitempty"` - NoticeResponse pgproto3.NoticeResponse `json:"notice_response,omitempty" yaml:"notice_response,omitempty"` - NotificationResponse pgproto3.NotificationResponse `json:"notification_response,omitempty" yaml:"notification_response,omitempty"` - ParameterDescription pgproto3.ParameterDescription `json:"parameter_description,omitempty" yaml:"parameter_description,omitempty"` - ParameterStatus pgproto3.ParameterStatus `yaml:"-"` - ParameterStatusCombined []pgproto3.ParameterStatus `json:"parameter_status,omitempty" yaml:"parameter_status,omitempty"` - ParseComplete pgproto3.ParseComplete `yaml:"-"` - ParseCompletes []pgproto3.ParseComplete `json:"parse_complete,omitempty" yaml:"parse_complete,omitempty"` - ReadyForQuery pgproto3.ReadyForQuery `json:"ready_for_query,omitempty" yaml:"ready_for_query,omitempty"` - RowDescription pgproto3.RowDescription `json:"row_description,omitempty" yaml:"row_description,omitempty,flow"` - PortalSuspended pgproto3.PortalSuspended `json:"portal_suspended,omitempty" yaml:"portal_suspended,omitempty"` - MsgType byte `json:"msg_type,omitempty" yaml:"msg_type,omitempty"` - AuthType int32 `json:"auth_type" yaml:"auth_type"` - // AuthMechanism string `json:"auth_mechanism,omitempty" yaml:"auth_mechanism,omitempty"` - BodyLen int `json:"body_len,omitempty" yaml:"body_len,omitempty"` -} - -type StartupPacket struct { - Length uint32 - ProtocolVersion uint32 -} - -type RegularPacket struct { - Identifier byte - Length uint32 - Payload []byte + // CopyBothResponse pgproto3.CopyBothResponse `json:"copy_both_response,omitempty" yaml:"copy_both_response,omitempty"` + CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` + // CopyInResponse pgproto3.CopyInResponse `json:"copy_in_response,omitempty" yaml:"copy_in_response,omitempty"` + // CopyOutResponse pgproto3.CopyOutResponse `json:"copy_out_response,omitempty" yaml:"copy_out_response,omitempty"` + CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` + DataRow pgproto3.DataRow `json:"-" yaml:"-"` + DataRows []pgproto3.DataRow `json:"data_row,omitempty" yaml:"data_row,omitempty,flow"` + EmptyQueryResponse pgproto3.EmptyQueryResponse `json:"empty_query_response,omitempty" yaml:"empty_query_response,omitempty"` + ErrorResponse pgproto3.ErrorResponse `json:"error_response,omitempty" yaml:"error_response,omitempty"` + FunctionCallResponse pgproto3.FunctionCallResponse `json:"function_call_response,omitempty" yaml:"function_call_response,omitempty"` + NoData pgproto3.NoData `json:"no_data,omitempty" yaml:"no_data,omitempty"` + NoticeResponse pgproto3.NoticeResponse `json:"notice_response,omitempty" yaml:"notice_response,omitempty"` + NotificationResponse pgproto3.NotificationResponse `json:"notification_response,omitempty" yaml:"notification_response,omitempty"` + ParameterDescription pgproto3.ParameterDescription `json:"parameter_description,omitempty" yaml:"parameter_description,omitempty"` + ParameterStatus pgproto3.ParameterStatus `json:"-" yaml:"-"` + ParameterStatusCombined []pgproto3.ParameterStatus `json:"parameter_status,omitempty" yaml:"parameter_status,omitempty"` + ParseComplete pgproto3.ParseComplete `json:"-" yaml:"-"` + ParseCompletes []pgproto3.ParseComplete `json:"parse_complete,omitempty" yaml:"parse_complete,omitempty"` + ReadyForQuery pgproto3.ReadyForQuery `json:"ready_for_query,omitempty" yaml:"ready_for_query,omitempty"` + RowDescription pgproto3.RowDescription `json:"row_description,omitempty" yaml:"row_description,omitempty,flow"` + PortalSuspended pgproto3.PortalSuspended `json:"portal_suspended,omitempty" yaml:"portal_suspended,omitempty"` + MsgType byte `json:"msg_type,omitempty" yaml:"msg_type,omitempty"` + AuthType int32 `json:"auth_type" yaml:"auth_type"` + BodyLen int `json:"body_len,omitempty" yaml:"body_len,omitempty"` } diff --git a/pkg/models/testbench.go b/pkg/models/testbench.go new file mode 100644 index 0000000000..74004074a5 --- /dev/null +++ b/pkg/models/testbench.go @@ -0,0 +1,12 @@ +package models + +type TestBenchReq struct { + KtclientID uint64 `json:"ktclientID"` + KtPid uint32 `json:"ktPid"` + KaPid uint32 `json:"kaPid"` +} + +type TestBenchResp struct { + IsSuccess bool `json:"isSuccess"` + Error string `json:"error"` +} diff --git a/pkg/platform/docker/docker.go b/pkg/platform/docker/docker.go index 88b1f57a4e..0328e9b00c 100644 --- a/pkg/platform/docker/docker.go +++ b/pkg/platform/docker/docker.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types" dockerContainerPkg "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" @@ -154,7 +153,7 @@ func (idc *Impl) StopAndRemoveDockerContainer() error { } } - removeOptions := types.ContainerRemoveOptions{ + removeOptions := dockerContainerPkg.RemoveOptions{ RemoveVolumes: true, Force: true, } @@ -175,7 +174,7 @@ func (idc *Impl) NetworkExists(networkName string) (bool, error) { defer cancel() // Retrieve all networks. - networks, err := idc.NetworkList(ctx, types.NetworkListOptions{}) + networks, err := idc.NetworkList(ctx, network.ListOptions{}) if err != nil { return false, fmt.Errorf("error retrieving networks: %v", err) } @@ -195,7 +194,7 @@ func (idc *Impl) CreateNetwork(networkName string) error { ctx, cancel := context.WithTimeout(context.Background(), idc.timeoutForDockerQuery) defer cancel() - _, err := idc.NetworkCreate(ctx, networkName, types.NetworkCreate{ + _, err := idc.NetworkCreate(ctx, networkName, network.CreateOptions{ Driver: "bridge", }) @@ -360,7 +359,7 @@ func (idc *Impl) GetHostWorkingDirectory() (string, error) { // Loop through container mounts and find the mount for current directory in the container for _, mount := range containerMounts { if mount.Destination == curDir { - idc.logger.Debug(fmt.Sprintf("found mount for %s in keploy-v2 container", curDir), zap.Any("mount", mount)) + idc.logger.Info(fmt.Sprintf("found mount for %s in keploy-v2 container", curDir), zap.Any("mount", mount)) return mount.Source, nil } } @@ -522,6 +521,42 @@ func (idc *Impl) SetKeployNetwork(c *Compose) (*NetworkInfo, error) { return networkInfo, nil } +func (idc *Impl) SetInitPid(c *Compose, containerName string) error { + for _, service := range c.Services.Content { + var containerNameMatch bool + var pidFound bool + + for i := 0; i < len(service.Content)-1; i++ { + if service.Content[i].Kind == yaml.ScalarNode && service.Content[i].Value == "container_name" && + service.Content[i+1].Kind == yaml.ScalarNode && service.Content[i+1].Value == containerName { + containerNameMatch = true + break + } + } + + if containerNameMatch { + for _, item := range service.Content { + if item.Value == "pid" { + pidFound = true + break + } + } + + // Add `pid: container:keploy-init` only if not already present + if !pidFound { + service.Content = append(service.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pid"}, + &yaml.Node{ + Kind: yaml.ScalarNode, + Value: "container:keploy-init", + }, + ) + } + } + } + return nil +} + // IsContainerRunning check if the container is already running or not, required for docker start command. func (idc *Impl) IsContainerRunning(containerName string) (bool, error) { diff --git a/pkg/platform/docker/service.go b/pkg/platform/docker/service.go index d41dea9263..7e22d13a4f 100644 --- a/pkg/platform/docker/service.go +++ b/pkg/platform/docker/service.go @@ -27,7 +27,7 @@ type Client interface { SetKeployNetwork(c *Compose) (*NetworkInfo, error) ReadComposeFile(filePath string) (*Compose, error) WriteComposeFile(compose *Compose, path string) error - + SetInitPid(c *Compose, containerName string) error IsContainerRunning(containerName string) (bool, error) CreateVolume(ctx context.Context, volumeName string, recreate bool) error } diff --git a/pkg/platform/docker/util.go b/pkg/platform/docker/util.go index f35f46ef73..0827154284 100644 --- a/pkg/platform/docker/util.go +++ b/pkg/platform/docker/util.go @@ -1,14 +1,34 @@ -//go:build linux +//go:build !windows package docker import ( + "context" + "errors" "fmt" + "os" + "os/exec" "regexp" + "runtime" + "strings" + "syscall" + "github.com/docker/docker/api/types/network" + "go.keploy.io/server/v2/config" "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/term" ) +type ConfigStruct struct { + DockerImage string + Envs map[string]string +} + +var DockerConfig = ConfigStruct{ + DockerImage: "ghcr.io/keploy/keploy", +} + func ParseDockerCmd(cmd string, kind utils.CmdType, idc Client) (string, string, error) { // Regular expression patterns @@ -51,3 +71,254 @@ func ParseDockerCmd(cmd string, kind utils.CmdType, idc Client) (string, string, return containerName, networkName, nil } + +func GenerateDockerEnvs(config ConfigStruct) string { + var envs []string + for key, value := range config.Envs { + envs = append(envs, fmt.Sprintf("-e %s='%s'", key, value)) + } + return strings.Join(envs, " ") +} + +// StartInDocker will check if the docker command is provided as an input +// then start the Keploy as a docker container and run the command +// should also return a boolean if the execution is moved to docker +func StartInDocker(ctx context.Context, logger *zap.Logger, conf *config.Config) error { + + if DockerConfig.Envs == nil { + DockerConfig.Envs = map[string]string{ + "INSTALLATION_ID": conf.InstallationID, + } + } else { + DockerConfig.Envs["INSTALLATION_ID"] = conf.InstallationID + } + + err := RunInDocker(ctx, logger) + if err != nil { + utils.LogError(logger, err, "failed to run the command in docker") + return err + } + // gracefully exit the current process + logger.Info("exiting the current process as the command is moved to docker") + os.Exit(0) + return nil +} + +func RunInDocker(ctx context.Context, logger *zap.Logger) error { + //Get the correct keploy alias. + keployAlias, err := getAlias(ctx, logger) + if err != nil { + return err + } + + client, err := New(logger) + if err != nil { + utils.LogError(logger, err, "failed to initalise docker") + return err + } + + addKeployNetwork(ctx, logger, client) + err = client.CreateVolume(ctx, "debugfs", true) + if err != nil { + utils.LogError(logger, err, "failed to debugfs volume") + return err + } + + var cmd *exec.Cmd + + // Detect the operating system + if runtime.GOOS == "windows" { + var args []string + args = append(args, "/C") + args = append(args, strings.Split(keployAlias, " ")...) + args = append(args, os.Args[1:]...) + // Use cmd.exe /C for Windows + cmd = exec.CommandContext( + ctx, + "cmd.exe", + args..., + ) + } else { + // Use sh -c for Unix-like systems + cmd = exec.CommandContext( + ctx, + "sh", + "-c", + keployAlias, + ) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + } + } + + cmd.Cancel = func() error { + err := utils.SendSignal(logger, -cmd.Process.Pid, syscall.SIGINT) + if err != nil { + utils.LogError(logger, err, "failed to start stop docker") + return err + } + return nil + } + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logger.Info("running the following command in docker", zap.String("command", cmd.String())) + err = cmd.Run() + if err != nil { + if ctx.Err() == context.Canceled { + return ctx.Err() + } + utils.LogError(logger, err, "failed to start keploy in docker") + return err + } + return nil +} + +func getAlias(ctx context.Context, logger *zap.Logger) (string, error) { + // Get the name of the operating system. + osName := runtime.GOOS + //TODO: configure the hardcoded port mapping + img := DockerConfig.DockerImage + ":v" + utils.Version + logger.Info("Starting keploy in docker with image", zap.String("image:", img)) + envs := GenerateDockerEnvs(DockerConfig) + if envs != "" { + envs = envs + " " + } + var ttyFlag string + + if term.IsTerminal(int(os.Stdin.Fd())) { + // ttyFlag = " -it " + ttyFlag = " " + } else { + ttyFlag = " " + } + + switch osName { + case "linux": + alias := "sudo docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + " -v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + case "windows": + // Get the current working directory + pwd, err := os.Getwd() + if err != nil { + utils.LogError(logger, err, "failed to get the current working directory") + } + dpwd := convertPathToUnixStyle(pwd) + cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") + out, err := cmd.Output() + if err != nil { + utils.LogError(logger, err, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] + if len(dockerContext) == 0 { + utils.LogError(logger, nil, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext = strings.Split(dockerContext, "\n")[0] + if dockerContext == "colima" { + logger.Info("Starting keploy in docker with colima context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img + return alias, nil + } + // if default docker context is used + logger.Info("Starting keploy in docker with default context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img + return alias, nil + case "darwin": + cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") + out, err := cmd.Output() + if err != nil { + utils.LogError(logger, err, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] + if len(dockerContext) == 0 { + utils.LogError(logger, nil, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext = strings.Split(dockerContext, "\n")[0] + if dockerContext == "colima" { + logger.Info("Starting keploy in docker with colima context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + } + // if default docker context is used + logger.Info("Starting keploy in docker with default context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + } + return "", errors.New("failed to get alias") +} + +func addKeployNetwork(ctx context.Context, logger *zap.Logger, client Client) { + + // Check if the 'keploy-network' network exists + networks, err := client.NetworkList(ctx, network.ListOptions{}) + if err != nil { + logger.Debug("failed to list docker networks") + return + } + + for _, network := range networks { + if network.Name == "keploy-network" { + logger.Debug("keploy network already exists") + return + } + } + + // Create the 'keploy' network if it doesn't exist + _, err = client.NetworkCreate(ctx, "keploy-network", network.CreateOptions{}) + if err != nil { + logger.Debug("failed to create keploy network") + return + } + + logger.Debug("keploy network created") +} + +func convertPathToUnixStyle(path string) string { + // Replace backslashes with forward slashes + unixPath := strings.Replace(path, "\\", "/", -1) + // Remove 'C:' + if len(unixPath) > 1 && unixPath[1] == ':' { + unixPath = unixPath[2:] + } + return unixPath +} + +// ExtractPidNamespaceInode extracts the inode of the PID namespace of a given PID +func ExtractPidNamespaceInode(pid int) (string, error) { + // Check the OS + if runtime.GOOS != "linux" { + // Execute command in the container to get the PID namespace + output, err := exec.Command("docker", "exec", "keploy-init", "stat", "/proc/1/ns/pid").Output() + if err != nil { + return "", err + } + outputStr := string(output) + + // Use a regular expression to extract the inode from the output + re := regexp.MustCompile(`pid:\[(\d+)\]`) + match := re.FindStringSubmatch(outputStr) + + if len(match) < 2 { + return "", fmt.Errorf("failed to extract PID namespace inode") + } + + pidNamespace := match[1] + return pidNamespace, nil + } + + // Check the namespace file in /proc + nsPath := fmt.Sprintf("/proc/%d/ns/pid", pid) + fileInfo, err := os.Stat(nsPath) + if err != nil { + return "", err + } + + // Retrieve inode number + inode := fileInfo.Sys().(*syscall.Stat_t).Ino + return fmt.Sprintf("%d", inode), nil +} diff --git a/pkg/platform/http/agent.go b/pkg/platform/http/agent.go new file mode 100644 index 0000000000..aa8558f6d0 --- /dev/null +++ b/pkg/platform/http/agent.go @@ -0,0 +1,765 @@ +//go:build !windows + +// Package http contains the client side code to communicate with the agent server +package http + +import ( + "bytes" + "context" + _ "embed" // necessary for embedding + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "sync" + "syscall" + "time" + + "github.com/docker/docker/api/types/events" + "go.keploy.io/server/v2/config" + "go.keploy.io/server/v2/pkg/agent/hooks" + "go.keploy.io/server/v2/pkg/client/app" + "go.keploy.io/server/v2/pkg/models" + kdocker "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type AgentClient struct { + logger *zap.Logger + dockerClient kdocker.Client //embedding the docker client to transfer the docker client methods to the core object + apps sync.Map + client http.Client + conf *config.Config +} + +//go:embed assets/initStop.sh +var initStopScript []byte + +func New(logger *zap.Logger, client kdocker.Client, c *config.Config) *AgentClient { + + return &AgentClient{ + logger: logger, + dockerClient: client, + client: http.Client{}, + conf: c, + } +} + +func (a *AgentClient) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + requestBody := models.IncomingReq{ + IncomingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for incoming request") + return nil, fmt.Errorf("error marshaling request body for incoming request: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/incoming", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for incoming request") + return nil, fmt.Errorf("error creating request for incoming request: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get incoming: %s", err.Error()) + } + + // Ensure response body is closed when we're done + go func() { + <-ctx.Done() + if res.Body != nil { + _ = res.Body.Close() + } + }() + + // Create a channel to stream TestCase data + tcChan := make(chan *models.TestCase) + + go func() { + defer close(tcChan) + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for incoming request") + } + }() + + decoder := json.NewDecoder(res.Body) + + for { + var testCase models.TestCase + if err := decoder.Decode(&testCase); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + // End of the stream + break + } + utils.LogError(a.logger, err, "failed to decode test case from stream") + break + } + + select { + case <-ctx.Done(): + // If the context is done, exit the loop + return + case tcChan <- &testCase: + // fmt.Println("Test case received for client", id, "TESTCASE", testCase) + // Send the decoded test case to the channel + } + } + }() + + return tcChan, nil +} + +func (a *AgentClient) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { + requestBody := models.OutgoingReq{ + OutgoingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for mock outgoing") + return nil, fmt.Errorf("error marshaling request body for mock outgoing: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/outgoing", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for mock outgoing") + return nil, fmt.Errorf("error creating request for mock outgoing: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get outgoing response: %s", err.Error()) + } + + mockChan := make(chan *models.Mock) + + grp, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return nil, fmt.Errorf("failed to get errorgroup from the context") + } + + grp.Go(func() error { + defer close(mockChan) + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for getoutgoing") + } + }() + + decoder := json.NewDecoder(res.Body) + + for { + var mock models.Mock + if err := decoder.Decode(&mock); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + // End of the stream + break + } + utils.LogError(a.logger, err, "failed to decode mock from stream") + break + } + + select { + case <-ctx.Done(): + // If the context is done, exit the loop + return nil + case mockChan <- &mock: + // Send the decoded mock to the channel + } + } + return nil + }) + + return mockChan, nil +} + +func (a *AgentClient) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { + // make a request to the server to mock outgoing + requestBody := models.OutgoingReq{ + OutgoingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for mock outgoing") + return fmt.Errorf("error marshaling request body for mock outgoing: %s", err.Error()) + } + + // mock outgoing request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/mock", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for mock outgoing") + return fmt.Errorf("error creating request for mock outgoing: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request for mockOutgoing: %s", err.Error()) + } + + var mockResp models.AgentResp + err = json.NewDecoder(res.Body).Decode(&mockResp) + if err != nil { + return fmt.Errorf("failed to decode response body for mock outgoing: %s", err.Error()) + } + + if mockResp.Error != nil { + return mockResp.Error + } + + return nil + +} + +func (a *AgentClient) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { + requestBody := models.SetMocksReq{ + Filtered: filtered, + UnFiltered: unFiltered, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for setmocks") + return fmt.Errorf("error marshaling request body for setmocks: %s", err.Error()) + } + + // mock outgoing request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/setmocks", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for setmocks outgoing") + return fmt.Errorf("error creating request for set mocks: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request for setmocks: %s", err.Error()) + } + + var mockResp models.AgentResp + err = json.NewDecoder(res.Body).Decode(&mockResp) + if err != nil { + return fmt.Errorf("failed to decode response body for setmocks: %s", err.Error()) + } + + if mockResp.Error != nil { + return mockResp.Error + } + + return nil +} + +func (a *AgentClient) GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) { + // Create the URL with query parameters + url := fmt.Sprintf("http://localhost:%d/agent/consumedmocks?id=%d", a.conf.Agent.Port, id) + + // Create a new GET request with the query parameter + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %s", err.Error()) + } + + req.Header.Set("Content-Type", "application/json") + + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request for mockOutgoing: %s", err.Error()) + } + + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for getconsumedmocks") + } + }() + + var consumedMocks []string + err = json.NewDecoder(res.Body).Decode(&consumedMocks) + if err != nil { + return nil, fmt.Errorf("failed to decode response body: %s", err.Error()) + } + + return consumedMocks, nil +} + +func (a *AgentClient) GetContainerIP(_ context.Context, clientID uint64) (string, error) { + + app, err := a.getApp(clientID) + if err != nil { + utils.LogError(a.logger, err, "failed to get app") + return "", err + } + + ip := app.ContainerIPv4Addr() + a.logger.Debug("ip address of the target app container", zap.Any("ip", ip)) + if ip == "" { + return "", fmt.Errorf("failed to get the IP address of the app container. Try increasing --delay (in seconds)") + } + + return ip, nil +} + +func (a *AgentClient) Run(ctx context.Context, clientID uint64, _ models.RunOptions) models.AppError { + + app, err := a.getApp(clientID) + if err != nil { + utils.LogError(a.logger, err, "failed to get app while running") + return models.AppError{AppErrorType: models.ErrInternal, Err: err} + } + + runAppErrGrp, runAppCtx := errgroup.WithContext(ctx) + + appErrCh := make(chan models.AppError, 1) + + defer func() { + err := runAppErrGrp.Wait() + + if err != nil { + utils.LogError(a.logger, err, "failed to stop the app") + } + }() + + runAppErrGrp.Go(func() error { + defer utils.Recover(a.logger) + defer close(appErrCh) + appErr := app.Run(runAppCtx) + if appErr.Err != nil { + utils.LogError(a.logger, appErr.Err, "error while running the app") + appErrCh <- appErr + } + return nil + }) + + select { + case <-runAppCtx.Done(): + return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: nil} + case appErr := <-appErrCh: + return appErr + } +} + +func (a *AgentClient) Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) { + + clientID := utils.GenerateID() + isDockerCmd := utils.IsDockerCmd(utils.CmdType(opts.CommandType)) + + // check if the agent is running + isAgentRunning := a.isAgentRunning(ctx) + if opts.EnableTesting { + isAgentRunning = false + // get the client pid of the test client and send it to the record agent + } + + if !isAgentRunning { + // Start the keploy agent as a detached process and pipe the logs into a file + if !isDockerCmd && runtime.GOOS != "linux" { + return 0, fmt.Errorf("Operating system not supported for this feature") + } + + if isDockerCmd { + // run the docker container instead of the agent binary + go func() { + if err := a.StartInDocker(ctx, a.logger); err != nil && !errors.Is(ctx.Err(), context.Canceled) { + a.logger.Error("failed to start Docker agent", zap.Error(err)) + } + }() + } else { + // Open the log file in append mode or create it if it doesn't exist + filepath := fmt.Sprintf("keploy_agent_%d.log", clientID) + logFile, err := os.OpenFile(filepath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + utils.LogError(a.logger, err, "failed to open log file") + return 0, err + } + + defer func() { + err := logFile.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close agent log file") + } + }() + + keployBin, err := utils.GetCurrentBinaryPath() + + if err != nil { + utils.LogError(a.logger, err, "failed to get current keploy binary path") + return 0, err + } + agentCmd := exec.Command("sudo", keployBin, "agent", "--port", strconv.Itoa(int(a.conf.ServerPort)), "--proxy-port", strconv.Itoa(int(a.conf.ProxyPort)), "--enable-testing", strconv.FormatBool(opts.EnableTesting)) + agentCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} // Detach the process + + // Redirect the standard output and error to the log file + agentCmd.Stdout = logFile + agentCmd.Stderr = logFile + + if err := agentCmd.Start(); err != nil { + utils.LogError(a.logger, err, "failed to start keploy agent") + return 0, err + } + + if opts.Mode == models.MODE_TEST && opts.EnableTesting { + a.SendKtPID(ctx, clientID) + } + a.logger.Info("keploy agent started", zap.Int("pid", agentCmd.Process.Pid)) + } + } + + // Channel to monitor if the agent is up and running + runningChan := make(chan bool) + + go func() { + for { + select { + case <-ctx.Done(): + // If the context is canceled, close the channel and return immediately + close(runningChan) + return + default: + if a.isAgentRunning(ctx) { + runningChan <- true + return + } + time.Sleep(1 * time.Second) // Poll every second + } + } + }() + + // Wait until the agent is ready or context is canceled + select { + case <-ctx.Done(): + // Handle context cancellation gracefully if Ctrl+C is pressed + a.logger.Info("Setup was canceled before the agent became ready") + return 0, fmt.Errorf("setup canceled before agent startup") + case <-runningChan: + // Proceed with setup if the agent becomes ready + a.logger.Info("Agent is now running, proceeding with setup") + } + + // Continue with app setup and registration as per normal flow + usrApp := app.NewApp(a.logger, clientID, cmd, a.dockerClient, app.Options{ + DockerNetwork: opts.DockerNetwork, + Container: opts.Container, + DockerDelay: opts.DockerDelay, + }) + a.apps.Store(clientID, usrApp) + + err := usrApp.Setup(ctx) + if err != nil { + utils.LogError(a.logger, err, "failed to setup app") + return 0, err + } + + if isDockerCmd { + opts.DockerNetwork = usrApp.KeployNetwork + inode, err := a.Initcontainer(ctx, app.Options{ + DockerNetwork: opts.DockerNetwork, + Container: opts.Container, + DockerDelay: opts.DockerDelay, + }) + if err != nil { + utils.LogError(a.logger, err, "failed to setup init container") + return 0, err + } + opts.AppInode = inode + } + + opts.ClientID = clientID + if err := a.RegisterClient(ctx, opts); err != nil { + utils.LogError(a.logger, err, "failed to register client") + return 0, err + } + isAgentRunning = a.isAgentRunning(ctx) + if !isAgentRunning { + return 0, fmt.Errorf("keploy agent is not running, please start the agent first") + } + + return clientID, nil +} + +func (a *AgentClient) getApp(clientID uint64) (*app.App, error) { + ap, ok := a.apps.Load(clientID) + if !ok { + return nil, fmt.Errorf("app with id:%v not found", clientID) + } + + // type assertion on the app + h, ok := ap.(*app.App) + if !ok { + return nil, fmt.Errorf("failed to type assert app with id:%v", clientID) + } + + return h, nil +} + +// RegisterClient registers the client with the server +func (a *AgentClient) RegisterClient(ctx context.Context, opts models.SetupOptions) error { + + isAgent := a.isAgentRunning(ctx) + if !isAgent { + return fmt.Errorf("keploy agent is not running, please start the agent first") + } + + // Register the client with the server + clientPid := uint32(os.Getpid()) + + // start the app container and get the inode number + // keploy agent would have already runnning, + var inode uint64 + var err error + if runtime.GOOS == "linux" { + // send the network info to the kernel + inode, err = hooks.GetSelfInodeNumber() + if err != nil { + a.logger.Error("failed to get inode number") + } + } + + // Register the client with the server + requestBody := models.RegisterReq{ + SetupOptions: models.SetupOptions{ + DockerNetwork: opts.DockerNetwork, + ClientNsPid: clientPid, + Mode: opts.Mode, + ClientID: opts.ClientID, + ClientInode: inode, + IsDocker: a.conf.Agent.IsDocker, + AppInode: opts.AppInode, + ProxyPort: a.conf.ProxyPort, + }, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for register client") + return fmt.Errorf("error marshaling request body for register client: %s", err.Error()) + } + + resp, err := a.client.Post(fmt.Sprintf("http://localhost:%d/agent/register", a.conf.Agent.Port), "application/json", bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to send register client request") + return fmt.Errorf("error sending register client request: %s", err.Error()) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to register client: %s", resp.Status) + } + + a.logger.Info("Client registered successfully with clientId", zap.Uint64("clientID", opts.ClientID)) + + // TODO: Read the response body in which we return the app id + var RegisterResp models.AgentResp + err = json.NewDecoder(resp.Body).Decode(&RegisterResp) + if err != nil { + utils.LogError(a.logger, err, "failed to decode response body for register client") + return fmt.Errorf("error decoding response body for register client: %s", err.Error()) + } + + if RegisterResp.Error != nil { + return RegisterResp.Error + } + + return nil +} + +func (a *AgentClient) UnregisterClient(ctx context.Context, unregister models.UnregisterReq) error { + // Unregister the client with the server + isAgentRunning := a.isAgentRunning(context.Background()) + if !isAgentRunning { + a.logger.Warn("keploy agent is not running, skipping unregister client") + return io.EOF + } + + fmt.Println("Unregistering the client with the server") + requestJSON, err := json.Marshal(unregister) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for unregister client") + return fmt.Errorf("error marshaling request body for unregister client: %s", err.Error()) + } + + // Passed background context as we dont want to cancel the unregister request upon client ctx cancellation + req, err := http.NewRequestWithContext(context.Background(), "POST", fmt.Sprintf("http://localhost:%d/agent/unregister", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for unregister client") + return fmt.Errorf("error creating request for unregister client: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := a.client.Do(req) + if err != nil && err != io.EOF { + return fmt.Errorf("failed to send request for unregister client: %s", err.Error()) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to unregister client: %s", resp.Status) + } + + return nil +} + +func (a *AgentClient) StartInDocker(ctx context.Context, logger *zap.Logger) error { + // Start the Keploy agent in a Docker container, directly using the passed context for cancellation + agentCtx := context.WithoutCancel(ctx) + err := kdocker.StartInDocker(agentCtx, logger, &config.Config{ + InstallationID: a.conf.InstallationID, + }) + if err != nil { + utils.LogError(logger, err, "failed to start keploy agent in docker") + return err + } + return nil +} + +func (a *AgentClient) Initcontainer(ctx context.Context, opts app.Options) (uint64, error) { + // Create a temporary file for the embedded initStop.sh script + initFile, err := os.CreateTemp("", "initStop.sh") + if err != nil { + a.logger.Error("failed to create temporary file", zap.Error(err)) + return 0, err + } + defer func() { + err := os.Remove(initFile.Name()) + if err != nil { + a.logger.Error("failed to remove temporary file", zap.Error(err)) + } + }() + + _, err = initFile.Write(initStopScript) + if err != nil { + a.logger.Error("failed to write script to temporary file", zap.Error(err)) + return 0, err + } + + // Close the file after writing to avoid 'text file busy' error + if err := initFile.Close(); err != nil { + a.logger.Error("failed to close temporary file", zap.Error(err)) + return 0, err + } + + err = os.Chmod(initFile.Name(), 0755) + if err != nil { + a.logger.Error("failed to make temporary script executable", zap.Error(err)) + return 0, err + } + + // Create a channel to signal when the container starts + containerStarted := make(chan struct{}) + + // Start the Docker events listener in a separate goroutine + go func() { + events, errs := a.dockerClient.Events(ctx, events.ListOptions{}) + for { + select { + case event := <-events: + if event.Type == "container" && event.Action == "start" && event.Actor.Attributes["name"] == "keploy-init" { + a.logger.Info("Container keploy-init started") + containerStarted <- struct{}{} + return + } + case err := <-errs: + a.logger.Error("Error while listening to Docker events", zap.Error(err)) + return + case <-ctx.Done(): + return + } + } + }() + + // Start the init container to get the PID namespace inode + cmdCancel := func(cmd *exec.Cmd) func() error { + return func() error { + a.logger.Info("sending SIGINT to the Initcontainer", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) + err := utils.SendSignal(a.logger, -cmd.Process.Pid, syscall.SIGINT) + return err + } + } + + cmd := fmt.Sprintf("docker run --network=%s --name keploy-init --rm -v%s:/initStop.sh alpine /initStop.sh", opts.DockerNetwork, initFile.Name()) + + // Execute the command + grp, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return 0, fmt.Errorf("failed to get errorgroup from the context") + } + + grp.Go(func() error { + println("Executing the init container command") + cmdErr := utils.ExecuteCommand(ctx, a.logger, cmd, cmdCancel, 25*time.Second) + if cmdErr.Err != nil && cmdErr.Type == utils.Init { + utils.LogError(a.logger, cmdErr.Err, "failed to execute init container command") + } + + println("Init container stopped") + return nil + }) + + // Wait for the container to start or context to cancel + select { + case <-containerStarted: + a.logger.Info("keploy-init container is running") + case <-ctx.Done(): + return 0, fmt.Errorf("context canceled while waiting for container to start") + } + + // Get the PID of the container's first process + inspect, err := a.dockerClient.ContainerInspect(ctx, "keploy-init") + if err != nil { + a.logger.Error("failed to inspect container", zap.Error(err)) + return 0, err + } + + pid := inspect.State.Pid + a.logger.Info("Container PID", zap.Int("pid", pid)) + + // Extract inode from the PID namespace + pidNamespaceInode, err := kdocker.ExtractPidNamespaceInode(pid) + if err != nil { + a.logger.Error("failed to extract PID namespace inode", zap.Error(err)) + return 0, err + } + + a.logger.Info("PID Namespace Inode", zap.String("inode", pidNamespaceInode)) + iNode, err := strconv.ParseUint(pidNamespaceInode, 10, 64) + if err != nil { + a.logger.Error("failed to convert inode to uint64", zap.Error(err)) + return 0, err + } + return iNode, nil +} + +func (a *AgentClient) isAgentRunning(ctx context.Context) bool { + + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://localhost:%d/agent/health", a.conf.Agent.Port), nil) + if err != nil { + utils.LogError(a.logger, err, "failed to send request to the agent server") + } + resp, err := a.client.Do(req) + if err != nil { + a.logger.Info("Keploy agent is not running in background, starting the agent") + return false + } + a.logger.Info("Setup request sent to the server", zap.String("status", resp.Status)) + return true +} diff --git a/pkg/platform/http/assets/initStop.sh b/pkg/platform/http/assets/initStop.sh new file mode 100755 index 0000000000..87bb6bd30a --- /dev/null +++ b/pkg/platform/http/assets/initStop.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Handle SIGINT and SIGTERM signals, forwarding them to the sleep process +trap 'echo "Init Container received SIGTERM or SIGINT, exiting..."; exit' SIGINT SIGTERM + +# Start sleep infinity to keep the container running in the background +sleep infinity & + +# Wait for the background process and forward any signals +wait $! diff --git a/pkg/platform/http/testBench.go b/pkg/platform/http/testBench.go new file mode 100644 index 0000000000..4849d0bb5e --- /dev/null +++ b/pkg/platform/http/testBench.go @@ -0,0 +1,82 @@ +package http + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "go.keploy.io/server/v2/pkg/models" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" +) + +func (a *AgentClient) SendKtPID(ctx context.Context, id uint64) error { + a.logger.Info("Inside SendKtPID of agent binary !!", zap.Uint64("clientID", id)) + ktPid := uint32(os.Getpid()) + + time.Sleep(3 * time.Second) + //Extracting the pid of the keployTest agent + pid, err := GetPIDFromPort(8090) + if err != nil { + utils.LogError(a.logger, err, "failed to get the keployTest pid") + return fmt.Errorf("error getting keployTest pid: %s", err.Error()) + } + fmt.Println("KeployTest PID", pid) + + tb := models.TestBenchReq{ + KtclientID: id, + KtPid: ktPid, + KaPid: uint32(pid), + } + + // Marshal the ktPid to send it to the server + requestJSON, err := json.Marshal(tb) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for register client") + return fmt.Errorf("error marshaling request body for register client: %s", err.Error()) + } + + // Send the ktPid to the server + resp, err := a.client.Post("http://localhost:8086/agent/testbench", "application/json", bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to send register client request") + return fmt.Errorf("error sending register client request: %s", err.Error()) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to send keployTest Pid %s", resp.Status) + } + + return nil +} + +func GetPIDFromPort(port int) (int, error) { + cmd := exec.Command("sudo", "lsof", "-i", fmt.Sprintf(":%d", port)) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return 0, fmt.Errorf("error executing lsof command: %v", err) + } + + // Parse the output to find the PID + lines := strings.Split(out.String(), "\n") + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) > 1 && fields[1] != "PID" { + pid, err := strconv.Atoi(fields[1]) + if err != nil { + return 0, fmt.Errorf("error parsing PID: %v", err) + } + return pid, nil + } + } + return 0, fmt.Errorf("no process found on port %d", port) +} diff --git a/pkg/platform/yaml/configdb/user/db.go b/pkg/platform/yaml/configdb/user/db.go index 5dc3fae5fa..33b2091221 100644 --- a/pkg/platform/yaml/configdb/user/db.go +++ b/pkg/platform/yaml/configdb/user/db.go @@ -89,16 +89,13 @@ func New(logger *zap.Logger, cfg *config.Config) *Db { func (db *Db) GetInstallationID(_ context.Context) (string, error) { var id string var err error - inDocker := os.Getenv("KEPLOY_INDOCKER") - if inDocker == "true" { - id = os.Getenv("INSTALLATION_ID") - } else { - id, err = machineid.ID() - if err != nil { - db.logger.Debug("failed to get machine id", zap.Error(err)) - return "", nil - } + + id, err = machineid.ID() + if err != nil { + db.logger.Debug("failed to get machine id", zap.Error(err)) + return "", nil } + if id == "" { db.logger.Debug("got empty machine id") return "", nil diff --git a/pkg/platform/yaml/mockdb/util.go b/pkg/platform/yaml/mockdb/util.go index 31b3187c84..025e378c69 100644 --- a/pkg/platform/yaml/mockdb/util.go +++ b/pkg/platform/yaml/mockdb/util.go @@ -174,6 +174,7 @@ func EncodeMock(mock *models.Mock, logger *zap.Logger) (*yaml.NetworkTrafficDoc, return nil, err } default: + utils.LogError(logger, nil, "failed to marshal the recorded mock into yaml due to invalid kind of mock") return nil, errors.New("type of mock is invalid") } diff --git a/pkg/platform/yaml/utils.go b/pkg/platform/yaml/utils.go index 52367e57cc..3c70695c3b 100755 --- a/pkg/platform/yaml/utils.go +++ b/pkg/platform/yaml/utils.go @@ -171,7 +171,6 @@ func NewSessionIndex(path string, Logger *zap.Logger) (string, error) { } for _, v := range files { - // fmt.Println("name for the file", v.Name()) fileName := filepath.Base(v.Name()) fileNamePackets := strings.Split(fileName, "-") if len(fileNamePackets) == 3 { diff --git a/pkg/service/agent/agent.go b/pkg/service/agent/agent.go new file mode 100644 index 0000000000..b98cec832f --- /dev/null +++ b/pkg/service/agent/agent.go @@ -0,0 +1,298 @@ +//go:build linux + +// Package agent contains methods for setting up hooks and proxy along with registering keploy clients. +package agent + +import ( + "context" + "errors" + "fmt" + + "go.keploy.io/server/v2/pkg/agent" + "go.keploy.io/server/v2/pkg/agent/hooks" + "go.keploy.io/server/v2/pkg/agent/hooks/structs" + "go.keploy.io/server/v2/pkg/models" + kdocker "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type Agent struct { + logger *zap.Logger + agent.Proxy // embedding the Proxy interface to transfer the proxy methods to the core object + agent.Hooks // embedding the Hooks interface to transfer the hooks methods to the core object + agent.Tester // embedding the Tester interface to transfer the tester methods to the core object + dockerClient kdocker.Client //embedding the docker client to transfer the docker client methods to the core object + proxyStarted bool +} + +func New(logger *zap.Logger, hook agent.Hooks, proxy agent.Proxy, tester agent.Tester, client kdocker.Client) *Agent { + return &Agent{ + logger: logger, + Hooks: hook, + Proxy: proxy, + Tester: tester, + dockerClient: client, + } +} + +// Setup will create a new app and store it in the map, all the setup will be done here +func (a *Agent) Setup(ctx context.Context, opts models.SetupOptions) error { + a.logger.Info("Starting the agent in ", zap.String(string(opts.Mode), "mode")) + + err := a.Hook(ctx, 0, models.HookOptions{ + Mode: opts.Mode, + IsDocker: opts.IsDocker, + EnableTesting: opts.EnableTesting, + }) + if err != nil { + a.logger.Error("failed to hook into the app", zap.Error(err)) + } + + <-ctx.Done() + a.logger.Info("Context cancelled, stopping the agent") + return context.Canceled + +} + +func (a *Agent) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + return a.Hooks.Record(ctx, id, opts) +} + +func (a *Agent) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { + m := make(chan *models.Mock, 500) + + err := a.Proxy.Record(ctx, id, m, opts) + if err != nil { + return nil, err + } + + return m, nil +} + +func (a *Agent) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { + a.logger.Debug("Inside MockOutgoing of agent binary !!") + + err := a.Proxy.Mock(ctx, id, opts) + if err != nil { + return err + } + + return nil +} + +func (a *Agent) Hook(ctx context.Context, id uint64, opts models.HookOptions) error { + hookErr := errors.New("failed to hook into the app") + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // create a new error group for the hooks + hookErrGrp, _ := errgroup.WithContext(ctx) + hookCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the hookCtx to control the lifecycle of the hooks + hookCtx, hookCtxCancel := context.WithCancel(hookCtx) + hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) + + // create a new error group for the proxy + proxyErrGrp, _ := errgroup.WithContext(ctx) + proxyCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the proxyCtx to control the lifecycle of the proxy + proxyCtx, proxyCtxCancel := context.WithCancel(proxyCtx) + proxyCtx = context.WithValue(proxyCtx, models.ErrGroupKey, proxyErrGrp) + + hookErrGrp.Go(func() error { + <-ctx.Done() + + proxyCtxCancel() + err := proxyErrGrp.Wait() + if err != nil { + utils.LogError(a.logger, err, "failed to stop the proxy") + } + + hookCtxCancel() + err = hookErrGrp.Wait() + if err != nil { + utils.LogError(a.logger, err, "failed to unload the hooks") + } + return nil + }) + + // load hooks if the mode changes .. + err := a.Hooks.Load(hookCtx, id, agent.HookCfg{ + ClientID: id, + Pid: 0, + IsDocker: opts.IsDocker, + KeployIPV4: "172.18.0.2", + Mode: opts.Mode, + }) + + if err != nil { + utils.LogError(a.logger, err, "failed to load hooks") + return hookErr + } + + if a.proxyStarted { + a.logger.Info("Proxy already started") + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + err = a.Proxy.StartProxy(proxyCtx, agent.ProxyOptions{ + DNSIPv4Addr: "172.18.0.2", + //DnsIPv6Addr: "" + }) + if err != nil { + utils.LogError(a.logger, err, "failed to start proxy") + return hookErr + } + + a.proxyStarted = true + fmt.Println("opts.EnableTesting::", opts.EnableTesting) + if opts.EnableTesting { + // Setting up the test bench + err := a.Tester.Setup(ctx, models.TestingOptions{Mode: opts.Mode}) + if err != nil { + utils.LogError(a.logger, err, "error while setting up the test bench environment") + return errors.New("failed to setup the test bench") + } + } + + return nil +} + +func (a *Agent) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { + a.logger.Debug("Inside SetMocks of agent binary !!") + return a.Proxy.SetMocks(ctx, id, filtered, unFiltered) +} + +func (a *Agent) GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) { + return a.Proxy.GetConsumedMocks(ctx, id) +} + +func (a *Agent) DeRegisterClient(ctx context.Context, unregister models.UnregisterReq) error { + fmt.Println("Inside DeRegisterClient of agent binary !!") + // send the info of the mode if its test mode we dont need to send the last mock + + if unregister.Mode != models.MODE_TEST { + err := a.Proxy.MakeClientDeRegisterd(ctx) + if err != nil { + return err + } + } + err := a.Hooks.DeleteKeployClientInfo(unregister.ClientID) + if err != nil { + return err + } + + return nil +} + +func (a *Agent) RegisterClient(ctx context.Context, opts models.SetupOptions) error { + + a.logger.Info("Registering the client with the keploy server") + // send the network info to the kernel + err := a.SendNetworkInfo(ctx, opts) + if err != nil { + a.logger.Error("failed to send network info to the kernel", zap.Error(err)) + return err + } + + clientInfo := structs.ClientInfo{ + KeployClientNsPid: opts.ClientNsPid, + IsDockerApp: 0, + KeployClientInode: opts.ClientInode, + AppInode: opts.AppInode, + } + + switch opts.Mode { + case models.MODE_RECORD: + clientInfo.Mode = uint32(1) + case models.MODE_TEST: + clientInfo.Mode = uint32(2) + default: + clientInfo.Mode = uint32(0) + } + + if opts.IsDocker { + clientInfo.IsDockerApp = 1 + } + + return a.Hooks.SendKeployClientInfo(opts.ClientID, clientInfo) +} + +func (a *Agent) SendNetworkInfo(ctx context.Context, opts models.SetupOptions) error { + if !opts.IsDocker { + proxyIP, err := hooks.IPv4ToUint32("127.0.0.1") + if err != nil { + return err + } + proxyInfo := structs.ProxyInfo{ + IP4: proxyIP, + IP6: [4]uint32{0, 0, 0, 0}, + Port: opts.ProxyPort, + } + err = a.Hooks.SendClientProxyInfo(opts.ClientID, proxyInfo) + if err != nil { + return err + } + return nil + } + + inspect, err := a.dockerClient.ContainerInspect(ctx, "keploy-v2") + if err != nil { + utils.LogError(a.logger, nil, fmt.Sprintf("failed to get inspect keploy container:%v", inspect)) + return err + } + + keployNetworks := inspect.NetworkSettings.Networks + var keployIPv4 string + for n, settings := range keployNetworks { + if n == opts.DockerNetwork { + keployIPv4 = settings.IPAddress //keploy container IP + break + } + } + + ipv4, err := hooks.IPv4ToUint32(keployIPv4) + if err != nil { + return err + } + + var ipv6 [4]uint32 + if opts.IsDocker { + ipv6, err := hooks.ToIPv4MappedIPv6(keployIPv4) + if err != nil { + return fmt.Errorf("failed to convert ipv4:%v to ipv4 mapped ipv6 in docker env:%v", ipv4, err) + } + a.logger.Debug(fmt.Sprintf("IPv4-mapped IPv6 for %s is: %08x:%08x:%08x:%08x\n", keployIPv4, ipv6[0], ipv6[1], ipv6[2], ipv6[3])) + + } + + proxyInfo := structs.ProxyInfo{ + IP4: ipv4, + IP6: ipv6, + Port: 36789, + } + + err = a.Hooks.SendClientProxyInfo(opts.ClientID, proxyInfo) + if err != nil { + return err + } + return nil +} + +func (a *Agent) SendKtInfo(ctx context.Context, tb models.TestBenchReq) error { + tbInfo := structs.TestBenchInfo{ + KTestClientPID: tb.KtPid, + KTestAgentPID: tb.KaPid, + } + return a.Hooks.SendKtInfo(ctx, tbInfo) +} diff --git a/pkg/service/agent/service.go b/pkg/service/agent/service.go new file mode 100644 index 0000000000..fb3bc71640 --- /dev/null +++ b/pkg/service/agent/service.go @@ -0,0 +1,36 @@ +package agent + +import ( + "context" + + "go.keploy.io/server/v2/pkg/models" +) + +type Service interface { + Setup(ctx context.Context, opts models.SetupOptions) error + GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) + GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) + MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error + SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error + GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) + RegisterClient(ctx context.Context, opts models.SetupOptions) error + DeRegisterClient(ctx context.Context, opts models.UnregisterReq) error + SendKtInfo(ctx context.Context, tb models.TestBenchReq) error +} + +type Options struct { + // Platform Platform + Network string + Container string + SelfTesting bool + Mode models.Mode +} + +// type Platform string + +// var ( +// linux Platform = "linux" +// windows Platform = "windows" +// mac Platform = "mac" +// docker Platform = "docker" +// ) diff --git a/pkg/service/agent/utils.go b/pkg/service/agent/utils.go new file mode 100644 index 0000000000..d62565f354 --- /dev/null +++ b/pkg/service/agent/utils.go @@ -0,0 +1,2 @@ +// Package agent contains utilities for agent. +package agent diff --git a/pkg/service/orchestrator/orchestrator.go b/pkg/service/orchestrator/orchestrator.go index b07150fbde..7a44d018dd 100644 --- a/pkg/service/orchestrator/orchestrator.go +++ b/pkg/service/orchestrator/orchestrator.go @@ -1,5 +1,3 @@ -//go:build linux - // Package orchestrator acts as a main brain for both the record and replay services package orchestrator diff --git a/pkg/service/orchestrator/rerecord.go b/pkg/service/orchestrator/rerecord.go index 2f56eb9ccf..e968ddb10c 100644 --- a/pkg/service/orchestrator/rerecord.go +++ b/pkg/service/orchestrator/rerecord.go @@ -1,5 +1,3 @@ -//go:build linux - package orchestrator import ( @@ -207,7 +205,7 @@ func (o *Orchestrator) replayTests(ctx context.Context, testSet string) (bool, e if utils.IsDockerCmd(cmdType) { host = o.config.ContainerName - userIP, err = o.record.GetContainerIP(ctx, o.config.AppID) + userIP, err = o.replay.GetContainerIP(ctx, o.config.ClientID) if err != nil { utils.LogError(o.logger, err, "failed to get the app ip") return false, err diff --git a/pkg/service/record/record.go b/pkg/service/record/record.go index 78712fa56d..964bd55a01 100755 --- a/pkg/service/record/record.go +++ b/pkg/service/record/record.go @@ -1,5 +1,3 @@ -//go:build linux - // Package record provides functionality for recording and managing test cases and mocks. package record @@ -7,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "time" @@ -49,13 +48,15 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { runAppCtx := context.WithoutCancel(ctx) runAppCtx, runAppCtxCancel := context.WithCancel(runAppCtx) - hookErrGrp, _ := errgroup.WithContext(ctx) - hookCtx := context.WithoutCancel(ctx) - hookCtx, hookCtxCancel := context.WithCancel(hookCtx) - hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) - // reRecordCtx, reRecordCancel := context.WithCancel(ctx) - // defer reRecordCancel() // Cancel the context when the function returns + setupErrGrp, _ := errgroup.WithContext(ctx) + setupCtx := context.WithoutCancel(ctx) + _, setupCtxCancel := context.WithCancel(setupCtx) + setupCtx = context.WithValue(ctx, models.ErrGroupKey, setupErrGrp) + reqErrGrp, _ := errgroup.WithContext(ctx) + reqCtx := context.WithoutCancel(ctx) + _, reqCtxCancel := context.WithCancel(reqCtx) + reqCtx = context.WithValue(ctx, models.ErrGroupKey, reqErrGrp) var stopReason string // defining all the channels and variables required for the record @@ -63,7 +64,7 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { var appErrChan = make(chan models.AppError, 1) var insertTestErrChan = make(chan error, 10) var insertMockErrChan = make(chan error, 10) - var appID uint64 + var clientID uint64 var newTestSetID string var testCount = 0 var mockCountMap = make(map[string]int) @@ -80,26 +81,44 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { } } } + + unregister := models.UnregisterReq{ + ClientID: clientID, + Mode: models.MODE_RECORD, + } + + // Dont call the Unregister if there is an error in the running application + fmt.Println("app error type", runAppError.AppErrorType) + if runAppError.AppErrorType == "" { + err := r.instrumentation.UnregisterClient(ctx, unregister) + if err != nil && err != io.EOF { + fmt.Println("error in unregistering client record") + utils.LogError(r.logger, err, "failed to unregister client") + } + } + runAppCtxCancel() err := runAppErrGrp.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop application") } - hookCtxCancel() - err = hookErrGrp.Wait() + + setupCtxCancel() + err = setupErrGrp.Wait() if err != nil { - utils.LogError(r.logger, err, "failed to stop hooks") + utils.LogError(r.logger, err, "failed to stop setup execution, that covers init container") } + err = errGrp.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop recording") } + + reqCtxCancel() r.telemetry.RecordedTestSuite(newTestSetID, testCount, mockCountMap) }() defer close(appErrChan) - defer close(insertTestErrChan) - defer close(insertMockErrChan) newTestSetID, err := r.GetNextTestSetID(ctx) if err != nil { @@ -115,18 +134,17 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { default: } - // Instrument will setup the environment and start the hooks and proxy - appID, err = r.Instrument(hookCtx) + clientID, err = r.instrumentation.Setup(setupCtx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay, Mode: models.MODE_RECORD, CommandType: r.config.CommandType}) if err != nil { - stopReason = "failed to instrument the application" + stopReason = "failed setting up the environment" utils.LogError(r.logger, err, stopReason) return fmt.Errorf(stopReason) } - r.config.AppID = appID + r.config.ClientID = clientID // fetching test cases and mocks from the application and inserting them into the database - frames, err := r.GetTestAndMockChans(ctx, appID) + frames, err := r.GetTestAndMockChans(reqCtx, clientID) if err != nil { stopReason = "failed to get data frames" utils.LogError(r.logger, err, stopReason) @@ -155,6 +173,9 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { errGrp.Go(func() error { for mock := range frames.Outgoing { + if mock == nil || mock.GetKind() == "" { + continue + } err := r.mockDB.InsertMock(ctx, mock, newTestSetID) if err != nil { if ctx.Err() == context.Canceled { @@ -171,7 +192,7 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { // running the user application runAppErrGrp.Go(func() error { - runAppError = r.instrumentation.Run(runAppCtx, appID, models.RunOptions{}) + runAppError = r.instrumentation.Run(runAppCtx, clientID, models.RunOptions{}) if runAppError.AppErrorType == models.ErrCtxCanceled { return nil } @@ -233,64 +254,102 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { return fmt.Errorf(stopReason) } -func (r *Recorder) Instrument(ctx context.Context) (uint64, error) { - var stopReason string - - // setting up the environment for recording - appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay}) - if err != nil { - stopReason = "failed setting up the environment" - utils.LogError(r.logger, err, stopReason) - return 0, fmt.Errorf(stopReason) - } - r.config.AppID = appID - - // checking for context cancellation as we don't want to start the hooks and proxy if the context is cancelled - select { - case <-ctx.Done(): - return appID, nil - default: - // Starting the hooks and proxy - err = r.instrumentation.Hook(ctx, appID, models.HookOptions{Mode: models.MODE_RECORD, EnableTesting: r.config.EnableTesting, Rules: r.config.BypassRules}) - if err != nil { - stopReason = "failed to start the hooks and proxy" - utils.LogError(r.logger, err, stopReason) - if ctx.Err() == context.Canceled { - return appID, err - } - return appID, fmt.Errorf(stopReason) - } - } - return appID, nil -} - -func (r *Recorder) GetTestAndMockChans(ctx context.Context, appID uint64) (FrameChan, error) { +func (r *Recorder) GetTestAndMockChans(ctx context.Context, clientID uint64) (FrameChan, error) { incomingOpts := models.IncomingOptions{ Filters: r.config.Record.Filters, } - incomingChan, err := r.instrumentation.GetIncoming(ctx, appID, incomingOpts) - if err != nil { - return FrameChan{}, fmt.Errorf("failed to get incoming test cases: %w", err) - } outgoingOpts := models.OutgoingOptions{ Rules: r.config.BypassRules, MongoPassword: r.config.Test.MongoPassword, FallBackOnMiss: r.config.Test.FallBackOnMiss, } - outgoingChan, err := r.instrumentation.GetOutgoing(ctx, appID, outgoingOpts) - if err != nil { - return FrameChan{}, fmt.Errorf("failed to get outgoing mocks: %w", err) + + // Create channels to receive incoming and outgoing data + incomingChan := make(chan *models.TestCase) + outgoingChan := make(chan *models.Mock) + errChan := make(chan error, 2) + + g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return FrameChan{}, fmt.Errorf("failed to get error group from context") } + g.Go(func() error { + defer close(incomingChan) + + ch, err := r.instrumentation.GetIncoming(ctx, clientID, incomingOpts) + if err != nil { + errChan <- err + return fmt.Errorf("failed to get incoming test cases: %w", err) + } + for testCase := range ch { + incomingChan <- testCase + } + return nil + }) + + g.Go(func() error { + + defer close(outgoingChan) + mockReceived := false + // create a context without cancel + // change this name to some mockCtx error group + mockErrGrp, _ := errgroup.WithContext(ctx) + mockCtx := context.WithoutCancel(ctx) + mockCtx, mockCtxCancel := context.WithCancel(mockCtx) + + defer func() { + fmt.Println("closing reqCtx") + mockCtxCancel() + err := mockErrGrp.Wait() + if err != nil && err != io.EOF { + utils.LogError(r.logger, err, "failed to stop request execution") + } + }() + + // listen for ctx canecllation in a go routine + go func() { + select { + case <-ctx.Done(): + if !mockReceived { + fmt.Println("context cancelled in go routine") + mockCtxCancel() + } + } + }() + + ch, err := r.instrumentation.GetOutgoing(mockCtx, clientID, outgoingOpts) + if err != nil { + r.logger.Error("failed to get outgoing mocks", zap.Error(err)) + errChan <- err + return fmt.Errorf("failed to get outgoing mocks: %w", err) + } + + for mock := range ch { + mockReceived = true // Set flag if a mock is received + select { + case <-ctx.Done(): + if mock != nil { + fmt.Println("mock is not nil") + outgoingChan <- mock + } + return nil + default: + outgoingChan <- mock + } + } + return nil + }) + return FrameChan{ Incoming: incomingChan, Outgoing: outgoingChan, }, nil } -func (r *Recorder) RunApplication(ctx context.Context, appID uint64, opts models.RunOptions) models.AppError { - return r.instrumentation.Run(ctx, appID, opts) +func (r *Recorder) RunApplication(ctx context.Context, clientID uint64, opts models.RunOptions) models.AppError { + return r.instrumentation.Run(ctx, clientID, opts) } func (r *Recorder) GetNextTestSetID(ctx context.Context) (string, error) { @@ -301,6 +360,6 @@ func (r *Recorder) GetNextTestSetID(ctx context.Context) (string, error) { return pkg.NextID(testSetIDs, models.TestSetPattern), nil } -func (r *Recorder) GetContainerIP(ctx context.Context, id uint64) (string, error) { - return r.instrumentation.GetContainerIP(ctx, id) +func (r *Recorder) GetContainerIP(ctx context.Context, clientID uint64) (string, error) { + return r.instrumentation.GetContainerIP(ctx, clientID) } diff --git a/pkg/service/record/service.go b/pkg/service/record/service.go index 700ca53bf2..019913b961 100755 --- a/pkg/service/record/service.go +++ b/pkg/service/record/service.go @@ -10,17 +10,16 @@ type Instrumentation interface { //Setup prepares the environment for the recording Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) //Hook will load hooks and start the proxy server. - Hook(ctx context.Context, id uint64, opts models.HookOptions) error GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) // Run is blocking call and will execute until error Run(ctx context.Context, id uint64, opts models.RunOptions) models.AppError GetContainerIP(ctx context.Context, id uint64) (string, error) + UnregisterClient(ctx context.Context, opts models.UnregisterReq) error } type Service interface { Start(ctx context.Context, reRecord bool) error - GetContainerIP(ctx context.Context, id uint64) (string, error) } type TestDB interface { diff --git a/pkg/service/record/utils.go b/pkg/service/record/utils.go index aa96e8bc5c..6162eca075 100644 --- a/pkg/service/record/utils.go +++ b/pkg/service/record/utils.go @@ -1,3 +1 @@ -//go:build linux - package record diff --git a/pkg/service/replay/replay.go b/pkg/service/replay/replay.go index 37a6308bed..96502cb433 100644 --- a/pkg/service/replay/replay.go +++ b/pkg/service/replay/replay.go @@ -81,7 +81,11 @@ func (r *Replayer) Start(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) ctx = context.WithValue(ctx, models.ErrGroupKey, g) - var hookCancel context.CancelFunc + setupErrGrp, _ := errgroup.WithContext(ctx) + _ = context.WithoutCancel(ctx) + setupCtx := context.WithValue(ctx, models.ErrGroupKey, setupErrGrp) + setupCtx, setupCtxCancel := context.WithCancel(setupCtx) + var stopReason = "replay completed successfully" // defering the stop function to stop keploy in case of any error in record or in case of context cancellation @@ -90,12 +94,27 @@ func (r *Replayer) Start(ctx context.Context) error { case <-ctx.Done(): break default: + fmt.Println("stopping Keploy from test mode") + // unregister := models.UnregisterReq{ + // ClientID: r.config.ClientID, + // Mode: models.MODE_TEST, + // } + // err := r.instrumentation.UnregisterClient(ctx, unregister) + // if err != nil { + // fmt.Println("error in unregistering client replay") + // utils.LogError(r.logger, err, "failed to unregister client") + // } + time.Sleep(2 * time.Second) r.logger.Info("stopping Keploy", zap.String("reason", stopReason)) } - if hookCancel != nil { - hookCancel() + + setupCtxCancel() + err := setupErrGrp.Wait() + if err != nil { + utils.LogError(r.logger, err, "failed to stop setup execution, that covers init container") } - err := g.Wait() + + err = g.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop replaying") } @@ -171,8 +190,7 @@ func (r *Replayer) Start(ctx context.Context) error { } } - // Instrument will load the hooks and start the proxy - inst, err := r.Instrument(ctx) + inst, err := r.Instrument(setupCtx) if err != nil { stopReason = fmt.Sprintf("failed to instrument: %v", err) utils.LogError(r.logger, err, stopReason) @@ -182,8 +200,6 @@ func (r *Replayer) Start(ctx context.Context) error { return fmt.Errorf(stopReason) } - hookCancel = inst.HookCancel - var testSetResult bool testRunResult := true abortTestRun := false @@ -220,7 +236,7 @@ func (r *Replayer) Start(ctx context.Context) error { } } - testSetStatus, err := r.RunTestSet(ctx, testSet, testRunID, inst.AppID, false) + testSetStatus, err := r.RunTestSet(ctx, testSet, testRunID, inst.ClientID, false) if err != nil { stopReason = fmt.Sprintf("failed to run test set: %v", err) utils.LogError(r.logger, err, stopReason) @@ -331,33 +347,17 @@ func (r *Replayer) Instrument(ctx context.Context) (*InstrumentState, error) { r.logger.Info("Keploy will not mock the outgoing calls when base path is provided", zap.Any("base path", r.config.Test.BasePath)) return &InstrumentState{}, nil } - appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay}) + // Instrument will setup the environment and start the hooks and proxy + clientID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, CommandType: r.config.CommandType, DockerDelay: r.config.BuildDelay, Mode: models.MODE_TEST, EnableTesting: r.config.EnableTesting}) if err != nil { - if errors.Is(err, context.Canceled) { - return &InstrumentState{}, err - } - return &InstrumentState{}, fmt.Errorf("failed to setup instrumentation: %w", err) + stopReason := "failed setting up the environment" + utils.LogError(r.logger, err, stopReason) + return &InstrumentState{}, fmt.Errorf(stopReason) } - r.config.AppID = appID - var cancel context.CancelFunc - // starting the hooks and proxy - select { - case <-ctx.Done(): - return &InstrumentState{}, context.Canceled - default: - hookCtx := context.WithoutCancel(ctx) - hookCtx, cancel = context.WithCancel(hookCtx) - err = r.instrumentation.Hook(hookCtx, appID, models.HookOptions{Mode: models.MODE_TEST, EnableTesting: r.config.EnableTesting, Rules: r.config.BypassRules}) - if err != nil { - cancel() - if errors.Is(err, context.Canceled) { - return &InstrumentState{}, err - } - return &InstrumentState{}, fmt.Errorf("failed to start the hooks and proxy: %w", err) - } - } - return &InstrumentState{AppID: appID, HookCancel: cancel}, nil + r.config.ClientID = clientID + + return &InstrumentState{ClientID: clientID}, nil } func (r *Replayer) GetNextTestRunID(ctx context.Context) (string, error) { @@ -379,7 +379,7 @@ func (r *Replayer) GetTestCases(ctx context.Context, testID string) ([]*models.T return r.testDB.GetTestCases(ctx, testID) } -func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID string, appID uint64, serveTest bool) (models.TestSetStatus, error) { +func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID string, clientID uint64, serveTest bool) (models.TestSetStatus, error) { // creating error group to manage proper shutdown of all the go routines and to propagate the error to the caller runTestSetErrGrp, runTestSetCtx := errgroup.WithContext(ctx) @@ -475,7 +475,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s cmdType := utils.CmdType(r.config.CommandType) var userIP string - err = r.SetupOrUpdateMocks(runTestSetCtx, appID, testSetID, models.BaseTime, time.Now(), Start) + err = r.SetupOrUpdateMocks(runTestSetCtx, clientID, testSetID, models.BaseTime, time.Now(), Start) if err != nil { return models.TestSetStatusFailed, err } @@ -484,7 +484,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s if !serveTest { runTestSetErrGrp.Go(func() error { defer utils.Recover(r.logger) - appErr = r.RunApplication(runTestSetCtx, appID, models.RunOptions{}) + appErr = r.RunApplication(runTestSetCtx, clientID, models.RunOptions{}) if appErr.AppErrorType == models.ErrCtxCanceled { return nil } @@ -529,7 +529,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } if utils.IsDockerCmd(cmdType) { - userIP, err = r.instrumentation.GetContainerIP(ctx, appID) + userIP, err = r.instrumentation.GetContainerIP(ctx, clientID) if err != nil { return models.TestSetStatusFailed, err } @@ -617,7 +617,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s var loopErr error //No need to handle mocking when basepath is provided - err := r.SetupOrUpdateMocks(runTestSetCtx, appID, testSetID, testCase.HTTPReq.Timestamp, testCase.HTTPResp.Timestamp, Update) + err := r.SetupOrUpdateMocks(runTestSetCtx, clientID, testSetID, testCase.HTTPReq.Timestamp, testCase.HTTPResp.Timestamp, Update) if err != nil { utils.LogError(r.logger, err, "failed to update mocks") break @@ -646,7 +646,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s } started := time.Now().UTC() - resp, loopErr := HookImpl.SimulateRequest(runTestSetCtx, appID, testCase, testSetID) + resp, loopErr := HookImpl.SimulateRequest(runTestSetCtx, clientID, testCase, testSetID) if loopErr != nil { utils.LogError(r.logger, err, "failed to simulate request") failure++ @@ -655,7 +655,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s var consumedMocks []string if r.instrument { - consumedMocks, err = r.instrumentation.GetConsumedMocks(runTestSetCtx, appID) + consumedMocks, err = r.instrumentation.GetConsumedMocks(runTestSetCtx, clientID) if err != nil { utils.LogError(r.logger, err, "failed to get consumed filtered mocks") } @@ -889,6 +889,7 @@ func (r *Replayer) SetupOrUpdateMocks(ctx context.Context, appID uint64, testSet } } + // this will be sent to the proxy err = r.instrumentation.SetMocks(ctx, appID, filteredMocks, unfilteredMocks) if err != nil { utils.LogError(r.logger, err, "failed to set mocks") @@ -987,8 +988,9 @@ func (r *Replayer) printSummary(_ context.Context, _ bool) { } } -func (r *Replayer) RunApplication(ctx context.Context, appID uint64, opts models.RunOptions) models.AppError { - return r.instrumentation.Run(ctx, appID, opts) +func (r *Replayer) RunApplication(ctx context.Context, clientID uint64, opts models.RunOptions) models.AppError { + fmt.Println("Running the application with clientID: ", clientID) + return r.instrumentation.Run(ctx, clientID, opts) } func (r *Replayer) GetTestSetConf(ctx context.Context, testSet string) (*models.TestSet, error) { @@ -1142,3 +1144,7 @@ func (r *Replayer) DeleteTests(ctx context.Context, testSetID string, testCaseID func SetTestHooks(testHooks TestHooks) { HookImpl = testHooks } + +func (r *Replayer) GetContainerIP(ctx context.Context, id uint64) (string, error) { + return r.instrumentation.GetContainerIP(ctx, id) +} diff --git a/pkg/service/replay/service.go b/pkg/service/replay/service.go index 5ffa0414b1..87cddc2c7e 100644 --- a/pkg/service/replay/service.go +++ b/pkg/service/replay/service.go @@ -12,7 +12,7 @@ type Instrumentation interface { //Setup prepares the environment for the recording Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) //Hook will load hooks and start the proxy server. - Hook(ctx context.Context, id uint64, opts models.HookOptions) error + // Hook(ctx context.Context, id uint64, opts models.HookOptions) error MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error // SetMocks Allows for setting mocks between test runs for better filtering and matching SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error @@ -20,15 +20,16 @@ type Instrumentation interface { GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) // Run is blocking call and will execute until error Run(ctx context.Context, id uint64, opts models.RunOptions) models.AppError - + UnregisterClient(ctx context.Context, opts models.UnregisterReq) error GetContainerIP(ctx context.Context, id uint64) (string, error) } type Service interface { Start(ctx context.Context) error - Instrument(ctx context.Context) (*InstrumentState, error) + // Instrument(ctx context.Context) (*InstrumentState, error) GetNextTestRunID(ctx context.Context) (string, error) GetAllTestSetIDs(ctx context.Context) ([]string, error) + GetContainerIP(ctx context.Context, id uint64) (string, error) RunTestSet(ctx context.Context, testSetID string, testRunID string, appID uint64, serveTest bool) (models.TestSetStatus, error) GetTestSetStatus(ctx context.Context, testRunID string, testSetID string) (models.TestSetStatus, error) GetTestCases(ctx context.Context, testID string) ([]*models.TestCase, error) @@ -89,8 +90,7 @@ type Storage interface { } type InstrumentState struct { - AppID uint64 - HookCancel context.CancelFunc + ClientID uint64 } type MockAction string diff --git a/pkg/service/tools/tools.go b/pkg/service/tools/tools.go index d2db09a230..85c2215d0a 100644 --- a/pkg/service/tools/tools.go +++ b/pkg/service/tools/tools.go @@ -81,11 +81,19 @@ func (t *Tools) Update(ctx context.Context) error { t.logger.Info("Updating to Version: " + latestVersion) downloadURL := "" - if runtime.GOARCH == "amd64" { - downloadURL = "https://github.com/keploy/keploy/releases/latest/download/keploy_linux_amd64.tar.gz" - } else { - downloadURL = "https://github.com/keploy/keploy/releases/latest/download/keploy_linux_arm64.tar.gz" + + if runtime.GOOS == "linux" { + if runtime.GOARCH == "amd64" { + downloadURL = "https://github.com/keploy/keploy/releases/latest/download/keploy_linux_amd64.tar.gz" + } else { + downloadURL = "https://github.com/keploy/keploy/releases/latest/download/keploy_linux_arm64.tar.gz" + } } + + if runtime.GOOS == "darwin" { + downloadURL = "https://github.com/keploy/keploy/releases/latest/download/keploy_darwin_all.tar.gz" + } + err = t.downloadAndUpdate(ctx, t.logger, downloadURL) if err != nil { return err diff --git a/pkg/service/utgen/ai.go b/pkg/service/utgen/ai.go index 1468d85af0..ca623c5fb5 100644 --- a/pkg/service/utgen/ai.go +++ b/pkg/service/utgen/ai.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -37,11 +36,12 @@ type Prompt struct { } type CompletionParams struct { - Model string `json:"model"` - Messages []Message `json:"messages"` - MaxTokens int `json:"max_tokens"` - Stream bool `json:"stream"` - Temperature float32 `json:"temperature"` + Model string `json:"model"` + Messages []Message `json:"messages"` + MaxTokens int `json:"max_tokens,omitempty"` + MaxCompletionTokens int `json:"max_completion_tokens,omitempty"` + Stream *bool `json:"stream,omitempty"` + Temperature float32 `json:"temperature,omitempty"` } type Message struct { @@ -69,10 +69,6 @@ type ResponseChunk struct { Choices []Choice `json:"choices"` } -type Choice struct { - Delta Delta `json:"delta"` -} - type Delta struct { Content string `json:"content"` } @@ -87,9 +83,44 @@ type AIResponse struct { } type AIRequest struct { - MaxTokens int `json:"maxTokens"` - Prompt Prompt `json:"prompt"` - SessionID string `json:"sessionId"` + MaxTokens int `json:"maxTokens"` + Prompt Prompt `json:"prompt"` + SessionID string `json:"sessionId"` + Iteration int `json:"iteration"` + RequestPurpose PurposeType `json:"requestPurpose"` +} + +// PurposeType defines the type of purpose for the AI request. +type PurposeType string + +const ( + // TestForFunction represents a purpose type where the request is to test a function. + TestForFunction PurposeType = "TestForFunction" + + // TestForFile represents a purpose type where the request is to test a file. + TestForFile PurposeType = "TestForFile" +) + +type CompletionResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []Choice `json:"choices"` + Usage UsageData `json:"usage"` +} + +type Choice struct { + Index int `json:"index"` + FinishReason string `json:"finish_reason"` + Message Message `json:"message"` + Delta Delta `json:"delta"` +} + +type UsageData struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` } func NewAIClient(genConfig config.UtGen, apiKey, apiServerURL string, auth service.Auth, sessionID string, logger *zap.Logger) *AIClient { @@ -106,75 +137,46 @@ func NewAIClient(genConfig config.UtGen, apiKey, apiServerURL string, auth servi } } -func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (string, int, int, error) { +func (ai *AIClient) Call(ctx context.Context, completionParams CompletionParams, aiRequest AIRequest, stream bool) (string, error) { var apiBaseURL string - if prompt.System == "" && prompt.User == "" { - return "", 0, 0, errors.New("the prompt must contain 'system' and 'user' keys") - } - - var messages []Message - if prompt.System == "" { - messages = []Message{ - {Role: "user", Content: prompt.User}, - } - } else { - messages = []Message{ - {Role: "system", Content: prompt.System}, - {Role: "user", Content: prompt.User}, - } - } - - completionParams := CompletionParams{ - Model: ai.Model, - Messages: messages, - MaxTokens: maxTokens, - Stream: true, - Temperature: 0.2, - } - var apiKey string if ai.APIBase == ai.APIServerURL { token, err := ai.Auth.GetToken(ctx) if err != nil { - return "", 0, 0, fmt.Errorf("error getting token: %v", err) + return "", fmt.Errorf("error getting token: %v", err) } ai.Logger.Debug("Making AI request to API server", zap.String("api_server_url", ai.APIServerURL), zap.String("token", token)) httpClient := &http.Client{} - aiRequest := AIRequest{ - MaxTokens: maxTokens, - Prompt: *prompt, - SessionID: ai.SessionID, - } aiRequestBytes, err := json.Marshal(aiRequest) if err != nil { - return "", 0, 0, fmt.Errorf("error marshalling AI request: %v", err) + return "", fmt.Errorf("error marshalling AI request: %v", err) } req, err := http.NewRequest("POST", fmt.Sprintf("%s/ai/call", ai.APIServerURL), bytes.NewBuffer(aiRequestBytes)) if err != nil { - return "", 0, 0, fmt.Errorf("error creating request: %v", err) + return "", fmt.Errorf("error creating request: %v", err) } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+token) resp, err := httpClient.Do(req) if err != nil { - return "", 0, 0, fmt.Errorf("error making request: %v", err) + return "", fmt.Errorf("error making request: %v", err) } bodyBytes, _ := io.ReadAll(resp.Body) var aiResponse AIResponse err = json.Unmarshal(bodyBytes, &aiResponse) if err != nil { - return "", 0, 0, fmt.Errorf("error unmarshalling response body: %v", err) + return "", fmt.Errorf("error unmarshalling response body: %v", err) } if resp.StatusCode != http.StatusOK { - return "", 0, 0, fmt.Errorf("unexpected status code: %v, response body: %s", resp.StatusCode, aiResponse.Error) + return "", fmt.Errorf("unexpected status code: %v, response body: %s", resp.StatusCode, aiResponse.Error) } defer func() { err := resp.Body.Close() @@ -183,7 +185,7 @@ func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (st } }() - return aiResponse.FinalContent, aiResponse.PromptTokens, aiResponse.CompletionTokens, nil + return aiResponse.FinalContent, nil } else if ai.APIBase != "" { apiBaseURL = ai.APIBase } else { @@ -192,7 +194,7 @@ func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (st requestBody, err := json.Marshal(completionParams) if err != nil { - return "", 0, 0, fmt.Errorf("error marshalling request body: %v", err) + return "", fmt.Errorf("error marshalling request body: %v", err) } queryParams := "" @@ -202,7 +204,7 @@ func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (st req, err := http.NewRequestWithContext(ctx, "POST", apiBaseURL+"/chat/completions"+queryParams, bytes.NewBuffer(requestBody)) if err != nil { - return "", 0, 0, fmt.Errorf("error creating request: %v", err) + return "", fmt.Errorf("error creating request: %v", err) } if ai.APIKey == "" { @@ -218,7 +220,7 @@ func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (st client := &http.Client{} resp, err := client.Do(req) if err != nil { - return "", 0, 0, fmt.Errorf("error making request: %v", err) + return "", fmt.Errorf("error making request: %v", err) } defer func() { if err := resp.Body.Close(); err != nil { @@ -229,7 +231,7 @@ func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (st if resp.StatusCode != http.StatusOK { bodyBytes, _ := io.ReadAll(resp.Body) bodyString := string(bodyBytes) - return "", 0, 0, fmt.Errorf("unexpected status code: %v, response body: %s", resp.StatusCode, bodyString) + return "", fmt.Errorf("unexpected status code: %v, response body: %s", resp.StatusCode, bodyString) } var contentBuilder strings.Builder @@ -240,61 +242,75 @@ func (ai *AIClient) Call(ctx context.Context, prompt *Prompt, maxTokens int) (st } fmt.Println("Streaming results from LLM model...") + if stream { + for { + line, err := reader.ReadString('\n') + if err != nil && err != io.EOF { + utils.LogError(ai.Logger, err, "Error reading stream") + return "", err + } + line = strings.TrimSpace(strings.TrimPrefix(line, "data: ")) + if line == "[DONE]" { + break + } - for { - line, err := reader.ReadString('\n') - if err != nil && err != io.EOF { - utils.LogError(ai.Logger, err, "Error reading stream") - return "", 0, 0, err - } - line = strings.TrimSpace(strings.TrimPrefix(line, "data: ")) - if line == "[DONE]" { - break - } - - if line == "" { - continue - } + if line == "" { + continue + } - var chunk ModelResponse - err = json.Unmarshal([]byte(line), &chunk) - if err != nil { - utils.LogError(ai.Logger, err, "Error unmarshalling chunk") - continue - } + var chunk ModelResponse + err = json.Unmarshal([]byte(line), &chunk) + if err != nil { + utils.LogError(ai.Logger, err, "Error unmarshalling chunk") + continue + } - if len(chunk.Choices) > 0 { - if chunk.Choices[0].Delta != (Delta{}) { - contentBuilder.WriteString(chunk.Choices[0].Delta.Content) - if ai.Logger.Level() == zap.DebugLevel { - fmt.Print(chunk.Choices[0].Delta.Content) + if len(chunk.Choices) > 0 { + if chunk.Choices[0].Delta != (Delta{}) { + contentBuilder.WriteString(chunk.Choices[0].Delta.Content) + if ai.Logger.Level() == zap.DebugLevel { + fmt.Print(chunk.Choices[0].Delta.Content) + } } } - } - if err == io.EOF { - break + if err == io.EOF { + break + } + time.Sleep(10 * time.Millisecond) + } + } else { + responseData, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("error reading response: %v", err) + } + var completionResponse CompletionResponse + err = json.Unmarshal(responseData, &completionResponse) + if err != nil { + return "", fmt.Errorf("error unmarshalling response: %v", err) + } + if len(completionResponse.Choices) > 0 { + finalContent := completionResponse.Choices[0].Message.Content + return finalContent, nil } - time.Sleep(10 * time.Millisecond) - } - - if ai.Logger.Level() == zap.DebugLevel { - fmt.Println() } - finalContent := contentBuilder.String() - promptTokens := len(strings.Fields(prompt.System)) + len(strings.Fields(prompt.User)) - completionTokens := len(strings.Fields(finalContent)) - return finalContent, promptTokens, completionTokens, nil + return finalContent, nil } -func (ai *AIClient) SendCoverageUpdate(ctx context.Context, sessionID string, oldCoverage, newCoverage float64) error { +func (ai *AIClient) SendCoverageUpdate(ctx context.Context, sessionID string, oldCoverage, newCoverage float64, iterationCount int) error { // Construct the request body with session ID, old coverage, and new coverage + requestPurpose := TestForFile + if len(ai.FunctionUnderTest) > 0 { + requestPurpose = TestForFunction + } requestBody, err := json.Marshal(map[string]interface{}{ "sessionId": sessionID, "initalCoverage": oldCoverage, "finalCoverage": newCoverage, + "iteration": iterationCount, + "requestPurpose": requestPurpose, }) if err != nil { return fmt.Errorf("error marshalling request body: %v", err) diff --git a/pkg/service/utgen/assets/test_generation.toml b/pkg/service/utgen/assets/test_generation.toml index 51f2bbc56d..0aeea216d0 100644 --- a/pkg/service/utgen/assets/test_generation.toml +++ b/pkg/service/utgen/assets/test_generation.toml @@ -36,24 +36,22 @@ Your task is to generate additional unit tests to complement the existing test s - The primary objective is to **increase the overall code coverage significantly**. - Do not include the code coverage report or any policies in your response. -{{- if .function_under_test }} +{{ if .function_under_test }} - **Focus Function:** - You must generate test cases specifically targeting the function named `{{ .function_under_test }}`. - Ensure that the tests for this function cover all logic paths, edge cases, and error handling scenarios. {{ end }} -{{- if .additional_command }} +{{ if .additional_command }} - {{ .additional_command }} {{ end }} - ## Source File Here is the source file that you will be writing tests against, called `{{ .source_file_name }}`. Line numbers have been added for clarity and are not part of the original code. ========= {{ .source_file_numbered | trim }} ========= - ## Test File Here is the file that contains the existing tests, called `{{ .test_file_name }}`. ========= @@ -67,21 +65,13 @@ The following packages are already installed in the environment. Use these when {{ .installed_packages | trim }} ========= - -{%- if additional_includes_section | trim %} +{{ if .additional_includes_section | trim }} {{ .additional_includes_section | trim }} -{% endif %} - -{%- if failed_tests_section | trim %} +{{ end }} +{{ if .failed_tests_section | trim }} {{ .failed_tests_section | trim }} -{% endif %} - -{%- if additional_instructions_text | trim %} - -{{ .additional_instructions_text | trim }} -{% endif %} - +{{ end }} ## Code Coverage The following is the existing code coverage report. Use this to determine what tests to write, as you should only write tests that increase the overall coverage: @@ -95,11 +85,11 @@ The output must be a YAML object equivalent to type $NewTests, according to the ===== class SingleTest(BaseModel): test_behavior: str = Field(description="Short description of the behavior the test covers") -{%- if language in ["python","java"] %} - test_name: str = Field(description=" A short test name, in snake case, that reflects the behaviour to test") -{%- else %} - test_name: str = Field(description=" A short unique test name, that should reflect the test objective") -{%- endif %} +{{ if or (eq .language "python") (eq .language "java") }} + test_name: str = Field(description="A short test name, in snake case, that reflects the behaviour to test") +{{ else }} + test_name: str = Field(description="A short unique test name, that should reflect the test objective") +{{ end }} test_code: str = Field(description="A single test function, that tests the behavior described in 'test_behavior'. The test should be a written like its a part of the existing test suite, if there is one, and it can use existing helper functions, setup, or teardown code.") new_imports_code: str = Field(description="Code for new imports that are required for the new test function, and are not already present in the test file.") library_installation_code: str = Field(description="If new libraries are needed, specify the installation commands for each library separately.") @@ -119,40 +109,40 @@ existing_test_function_signature: | new_tests: - test_behavior: | Test that the function returns the correct output for a single element list -{%- if language in ["python","java"] %} +{{- if (or (eq .language "python") (eq .language "java")) }} test_name: | test_single_element_list -{%- else %} +{{- else }} test_name: | ... -{%- endif %} +{{- end }} test_code: | -{%- if language in ["python"] %} +{{- if eq .language "python" }} def ... -{%- else %} +{{- else }} ... -{%- endif %} +{{- end }} new_imports_code: | - {%- if language == "python" %} + {{- if eq .language "python" }} "import pytest" "from my_module import my_function" - {%- elif language == "java" %} + {{- else if eq .language "java" }} "import org.junit.jupiter.api.Test;" "import my.package.MyFunction;" - {%- elif language == "golang" %} + {{- else if eq .language "go" }} "import "testing" " "import "my_module"" - {%- elif language == "javascript" %} + {{- else if eq .language "javascript" }} "const assert = require('assert');" "const myFunction = require('my_module').myFunction;" - {%- elif language == "typescript" %} + {{- else if eq .language "typescript" }} "import { assert } from 'assert';" "import { myFunction } from 'my_module';" - {%- endif %} + {{- end }} library_installation_code: | - {%- if language == "python" %} + {{- if eq .language "python" }} pip install pytest - {%- elif language == "java" %} + {{- else if eq .language "java" }} # Add the following to your Maven pom.xml: # # org.junit.jupiter @@ -160,26 +150,24 @@ new_tests: # 5.7.0 # test # - {%- elif language == "golang" %} + {{- else if eq .language "go" }} go get github.com/my_module - {%- elif language == "javascript" %} + {{- else if eq .language "javascript" }} npm install assert - {%- elif language == "typescript" %} + {{- else if eq .language "typescript" }} npm install assert - {%- endif %} + {{- end }} test_tags: happy path ... ``` Use block scalar('|') to format each YAML output. - -{%- if additional_instructions_text| trim %} - -{{ .additional_instructions_text| trim }} -{% endif %} +{{- if .additional_instructions_text | trim }} +{{ .additional_instructions_text | trim }} +{{ end }} Response (should be a valid YAML, and nothing else): ```yaml -""" +""" \ No newline at end of file diff --git a/pkg/service/utgen/gen.go b/pkg/service/utgen/gen.go index f852eab8ef..3f87f6e3d1 100644 --- a/pkg/service/utgen/gen.go +++ b/pkg/service/utgen/gen.go @@ -213,12 +213,15 @@ func (g *UnitTestGenerator) Start(ctx context.Context) error { return err } g.failedTests = []*models.FailedUT{} - testsDetails, err := g.GenerateTests(ctx) + testsDetails, err := g.GenerateTests(ctx, iterationCount) if err != nil { utils.LogError(g.logger, err, "Error generating tests") return err } - + if testsDetails == nil { + g.logger.Info("No tests generated") + continue + } g.logger.Info("Validating new generated tests one by one") g.totalTestCase += len(testsDetails.NewTests) totalTest = len(testsDetails.NewTests) @@ -282,7 +285,7 @@ func (g *UnitTestGenerator) Start(ctx context.Context) error { fmt.Print(addHeightPadding(paddingHeight, 2, columnWidths2)) fmt.Printf("+------------------------------------------+------------------------------------------+\n") fmt.Printf("<=========================================>\n") - err = g.ai.SendCoverageUpdate(ctx, g.ai.SessionID, initialCoverage, g.cov.Current) + err = g.ai.SendCoverageUpdate(ctx, g.ai.SessionID, initialCoverage, g.cov.Current, iterationCount) if err != nil { utils.LogError(g.logger, err, "Error sending coverage update") } @@ -408,8 +411,8 @@ func (g *UnitTestGenerator) runCoverage() error { g.logger.Info(fmt.Sprintf("Test command completed in %v", formatDuration(duration))) if err != nil { - utils.LogError(g.logger, err, "Error running test command") - return fmt.Errorf("error running test command: %w", err) + g.logger.Warn("Test command failed. Ensure no tests are failing, and rerun the command.") + return fmt.Errorf("error running test command: %s", g.cmd) } if exitCode != 0 { utils.LogError(g.logger, err, "Error running test command") @@ -428,7 +431,7 @@ func (g *UnitTestGenerator) runCoverage() error { return nil } -func (g *UnitTestGenerator) GenerateTests(ctx context.Context) (*models.UTDetails, error) { +func (g *UnitTestGenerator) GenerateTests(ctx context.Context, iterationCount int) (*models.UTDetails, error) { fmt.Println("Generating Tests...") select { @@ -438,7 +441,18 @@ func (g *UnitTestGenerator) GenerateTests(ctx context.Context) (*models.UTDetail default: } - response, promptTokenCount, responseTokenCount, err := g.ai.Call(ctx, g.prompt, 4096) + requestPurpose := TestForFile + if len(g.ai.FunctionUnderTest) > 0 { + requestPurpose = TestForFunction + } + aiRequest := AIRequest{ + MaxTokens: 4096, + Prompt: *g.prompt, + SessionID: g.ai.SessionID, + Iteration: iterationCount, + RequestPurpose: requestPurpose, + } + response, err := g.ai.Call(ctx, CompletionParams{}, aiRequest, false) if err != nil { return &models.UTDetails{}, err } @@ -450,8 +464,6 @@ func (g *UnitTestGenerator) GenerateTests(ctx context.Context) (*models.UTDetail default: } - g.logger.Info(fmt.Sprintf("Total token used count for LLM model %s: %d", g.ai.Model, promptTokenCount+responseTokenCount)) - select { case <-ctx.Done(): err := ctx.Err() @@ -460,6 +472,7 @@ func (g *UnitTestGenerator) GenerateTests(ctx context.Context) (*models.UTDetail } testsDetails, err := unmarshalYamlTestDetails(response) + if err != nil { utils.LogError(g.logger, err, "Error unmarshalling test details") return &models.UTDetails{}, err @@ -500,7 +513,13 @@ func (g *UnitTestGenerator) getIndentation(ctx context.Context) (int, error) { if err != nil { return 0, fmt.Errorf("error building prompt: %w", err) } - response, _, _, err := g.ai.Call(ctx, prompt, 4096) + + aiRequest := AIRequest{ + MaxTokens: 4096, + Prompt: *prompt, + SessionID: g.ai.SessionID, + } + response, err := g.ai.Call(ctx, CompletionParams{}, aiRequest, false) if err != nil { utils.LogError(g.logger, err, "Error calling AI model") return 0, err @@ -531,7 +550,13 @@ func (g *UnitTestGenerator) getLine(ctx context.Context) (int, error) { if err != nil { return 0, fmt.Errorf("error building prompt: %w", err) } - response, _, _, err := g.ai.Call(ctx, prompt, 4096) + + aiRequest := AIRequest{ + MaxTokens: 4096, + Prompt: *prompt, + SessionID: g.ai.SessionID, + } + response, err := g.ai.Call(ctx, CompletionParams{}, aiRequest, false) if err != nil { utils.LogError(g.logger, err, "Error calling AI model") return 0, err diff --git a/pkg/service/utgen/injector.go b/pkg/service/utgen/injector.go index 0f9edda87d..dd44742d0e 100644 --- a/pkg/service/utgen/injector.go +++ b/pkg/service/utgen/injector.go @@ -191,9 +191,15 @@ func (i *Injector) updateJavaScriptImports(importedContent string, newImports [] } func (i *Injector) updateImports(filePath string, imports string) (int, error) { - newImports := strings.Split(imports, "\n") - for i, imp := range newImports { - newImports[i] = strings.TrimSpace(imp) + importLines := strings.Split(imports, "\n") + var newImports []string + + for _, imp := range importLines { + trimmedImp := strings.TrimSpace(imp) + if strings.Contains(trimmedImp, "No new imports") || strings.Contains(trimmedImp, "None") { + continue + } + newImports = append(newImports, trimmedImp) } contentBytes, err := os.ReadFile(filePath) if err != nil { diff --git a/pkg/service/utgen/prompt.go b/pkg/service/utgen/prompt.go index 35266a3fdc..760f889f8b 100644 --- a/pkg/service/utgen/prompt.go +++ b/pkg/service/utgen/prompt.go @@ -3,6 +3,7 @@ package utgen import ( "bytes" "fmt" + "html" "html/template" "os" "strings" @@ -165,6 +166,7 @@ func (pb *PromptBuilder) BuildPrompt(file, failedTestRuns string) (*Prompt, erro return prompt, fmt.Errorf("Error rendering user prompt: %v", err) } prompt.System = systemPrompt + userPrompt = html.UnescapeString(userPrompt) prompt.User = userPrompt return prompt, nil } diff --git a/utils/signal_others.go b/utils/signal_others.go index e450719c76..11ea93a542 100644 --- a/utils/signal_others.go +++ b/utils/signal_others.go @@ -29,15 +29,15 @@ func SendSignal(logger *zap.Logger, pid int, sig syscall.Signal) error { func ExecuteCommand(ctx context.Context, logger *zap.Logger, userCmd string, cancel func(cmd *exec.Cmd) func() error, waitDelay time.Duration) CmdError { // Run the app as the user who invoked sudo - username := os.Getenv("SUDO_USER") + // username := os.Getenv("SUDO_USER") cmd := exec.CommandContext(ctx, "sh", "-c", userCmd) - if username != "" { - // print all environment variables - logger.Debug("env inherited from the cmd", zap.Any("env", os.Environ())) - // Run the command as the user who invoked sudo to preserve the user environment variables and PATH - cmd = exec.CommandContext(ctx, "sudo", "-E", "-u", os.Getenv("SUDO_USER"), "env", "PATH="+os.Getenv("PATH"), "sh", "-c", userCmd) - } + // if username != "" { + // // print all environment variables + // logger.Debug("env inherited from the cmd", zap.Any("env", os.Environ())) + // // Run the command as the user who invoked sudo to preserve the user environment variables and PATH + // cmd = exec.CommandContext(ctx, "sudo", "-E", "-u", os.Getenv("SUDO_USER"), "env", "PATH="+os.Getenv("PATH"), "sh", "-c", userCmd) + // } // Set the cancel function for the command cmd.Cancel = cancel(cmd) @@ -53,7 +53,7 @@ func ExecuteCommand(ctx context.Context, logger *zap.Logger, userCmd string, can cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - logger.Debug("", zap.Any("executing cli", cmd.String())) + logger.Info("", zap.Any("executing cli", cmd.String())) err := cmd.Start() if err != nil { diff --git a/utils/utils.go b/utils/utils.go index 04a29ec606..39c1790a44 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "debug/elf" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -26,6 +27,7 @@ import ( "golang.org/x/text/language" "github.com/getsentry/sentry-go" + "github.com/google/uuid" netLib "github.com/shirou/gopsutil/v3/net" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -193,6 +195,16 @@ func DeleteFileIfNotExists(logger *zap.Logger, name string) (err error) { return nil } +func GenerateID() uint64 { + // Random AppId uint64 will be generated and maintain in a map and return the id to client + newUUID := uuid.New() + + // app id will be sent by the client. + // Convert the first 8 bytes of the UUID to an int64 + id := int64(binary.BigEndian.Uint64(newUUID[:8])) + return uint64(id) +} + type GitHubRelease struct { TagName string `json:"tag_name"` Body string `json:"body"` @@ -910,3 +922,18 @@ func IsFileEmpty(filePath string) (bool, error) { } return fileInfo.Size() == 0, nil } + +func GetCurrentBinaryPath() (string, error) { + executable, err := os.Executable() + if err != nil { + return "", err + } + + // Resolve the full path (to avoid issues with symbolic links) + executablePath, err := filepath.EvalSymlinks(executable) + if err != nil { + return "", err + } + + return executablePath, nil +}