diff --git a/.github/workflows/test_workflow_scripts/golang-docker.sh b/.github/workflows/test_workflow_scripts/golang-docker.sh index 618425eb0e..2eb6c89c40 100755 --- a/.github/workflows/test_workflow_scripts/golang-docker.sh +++ b/.github/workflows/test_workflow_scripts/golang-docker.sh @@ -22,10 +22,29 @@ docker build -t gin-mongo . docker rm -f ginApp 2>/dev/null || true container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -57,7 +76,11 @@ send_request(){ # Wait for 5 seconds for keploy to record the tcs and mocks. sleep 5 + # sudo docker rm -f keploy-v2 + # sleep 5 + # sudo docker rm -f keploy-init container_kill + # sleep 5 wait } @@ -69,32 +92,41 @@ for i in {1..2}; do if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 +# container_kill +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + +echo "Starting the test phase..." # Start the keploy in test mode. test_container="ginApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c 'docker run -p8080:8080 --net keploy-network --name ginApp_test gin-mongo' --containerName "$test_container" --apiTimeout 60 --delay 20 --generate-github-actions=false &> "${test_container}.txt" +# container_kill +# sudo docker rm -f keploy-v2 + if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/golang-linux.sh b/.github/workflows/test_workflow_scripts/golang-linux.sh index 42fccf5943..4ecbc355a9 100644 --- a/.github/workflows/test_workflow_scripts/golang-linux.sh +++ b/.github/workflows/test_workflow_scripts/golang-linux.sh @@ -26,9 +26,14 @@ sed -i 's/ports: 0/ports: 27017/' "$config_file" # Remove any preexisting keploy tests and mocks. rm -rf keploy/ +echo "Starting the pipeline..." + # Build the binary. go build -o ginApp +# Start keploy agent in the background + +echo "Keploy agent started" send_request(){ sleep 10 @@ -70,28 +75,37 @@ send_request(){ sudo kill $pid } - for i in {1..2}; do + echo "Starting iteration ${i}" app_name="javaApp_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & - sudo -E env PATH="$PATH" ./../../keployv2 record -c "./ginApp" &> "${app_name}.txt" + sudo -E env PATH="$PATH" ./../../keployv2 record -c "./ginApp" &> "${app_name}.txt" --debug if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done +sleep 10 +echo "Starting the pipeline for test mode..." + +sudo ./../../keployv2 agent & + +echo "Keploy agent started for test mode" + +sleep 10 + # Start the gin-mongo app in test mode. -sudo -E env PATH="$PATH" ./../../keployv2 test -c "./ginApp" --delay 7 &> test_logs.txt +sudo -E env PATH="$PATH" ./../../keployv2 test -c "./ginApp" --delay 7 &> test_logs.txt --debug if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." @@ -107,10 +121,8 @@ fi all_passed=true - # Get the test results from the testReport file. -for i in {0..1} -do +for i in {0..1}; do # Define the report file for each test set report_file="./keploy/reports/test-run-0/test-set-$i-report.yaml" @@ -135,4 +147,13 @@ if [ "$all_passed" = true ]; then else cat "test_logs.txt" exit 1 -fi \ No newline at end of file +fi + +# Finally, stop the keploy agent +agent_pid=$(pgrep -f 'keployv2 agent') +if [ -z "$agent_pid" ]; then + echo "Keploy agent process not found." +else + echo "Stopping keploy agent with PID: $agent_pid" + sudo kill $agent_pid +fi diff --git a/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh b/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh index 2033d1d687..56b6688047 100644 --- a/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh +++ b/.github/workflows/test_workflow_scripts/golang-mysql-linux.sh @@ -47,17 +47,19 @@ send_request() { for i in {1..2}; do app_name="urlShort_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH="$PATH" ./../../keployv2 record -c "./urlShort" --generateGithubActions=false &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait @@ -65,18 +67,20 @@ for i in {1..2}; do done # Start the gin-mongo app in test mode. +sudo ./../../keployv2 agent & +sleep 7 sudo -E env PATH="$PATH" ./../../keployv2 test -c "./urlShort" --delay 7 --generateGithubActions=false &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/java-linux.sh b/.github/workflows/test_workflow_scripts/java-linux.sh index 6150cbf45f..d39f887866 100644 --- a/.github/workflows/test_workflow_scripts/java-linux.sh +++ b/.github/workflows/test_workflow_scripts/java-linux.sh @@ -65,34 +65,38 @@ for i in {1..2}; do # Start keploy in record mode. mvn clean install -Dmaven.test.skip=true app_name="javaApp_${i}" + sudo ./../../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH=$PATH ./../../../keployv2 record -c 'java -jar target/spring-petclinic-rest-3.0.2.jar' &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done +sudo ./../../../keployv2 agent & +sleep 5 # Start keploy in test mode. sudo -E env PATH=$PATH ./../../../keployv2 test -c 'java -jar target/spring-petclinic-rest-3.0.2.jar' --delay 20 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/node-docker.sh b/.github/workflows/test_workflow_scripts/node-docker.sh index 6d21e70a7d..d32e06fc56 100755 --- a/.github/workflows/test_workflow_scripts/node-docker.sh +++ b/.github/workflows/test_workflow_scripts/node-docker.sh @@ -1,5 +1,6 @@ #!/bin/bash + source ./../../.github/workflows/test_workflow_scripts/test-iid.sh # Start the docker container. @@ -13,10 +14,29 @@ sudo rm -rf keploy/ docker build -t node-app:1.0 . container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -51,7 +71,7 @@ send_request(){ curl -X GET http://localhost:8000/students # Wait for 5 seconds for keploy to record the tcs and mocks. - sleep 5 + sleep 10 container_kill wait } @@ -64,31 +84,39 @@ for i in {1..2}; do if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 + +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + +echo "Starting the test phase..." + # Start keploy in test mode. test_container="nodeApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c "docker run -p8000:8000 --rm --name $test_container --network keploy-network node-app:1.0" --containerName "$test_container" --apiTimeout 30 --delay 30 --generate-github-actions=false &> "${test_container}.txt" + if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi # Monitor Docker logs for race conditions during testing. if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/node-linux.sh b/.github/workflows/test_workflow_scripts/node-linux.sh index 5c4e3f2184..85c0d7014c 100755 --- a/.github/workflows/test_workflow_scripts/node-linux.sh +++ b/.github/workflows/test_workflow_scripts/node-linux.sh @@ -47,17 +47,19 @@ send_request(){ # Record and test sessions in a loop for i in {1..2}; do app_name="nodeApp_${i}" + sudo ./../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH=$PATH ./../../keployv2 record -c 'npm start' &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait @@ -67,6 +69,8 @@ done mocks_file="keploy/test-set-0/tests/test-5.yaml" sed -i 's/"page":1/"page":4/' "$mocks_file" +sudo ./../../keployv2 agent & +sleep 5 # Test modes and result checking sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 &> test_logs1.txt @@ -81,30 +85,34 @@ if grep "WARNING: DATA RACE" "test_logs1.txt"; then exit 1 fi +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --delay 10 --testsets test-set-0 &> test_logs2.txt if grep "ERROR" "test_logs2.txt"; then echo "Error found in pipeline..." cat "test_logs2.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs2.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs2.txt" - exit 1 + # exit 1 fi sed -i 's/selectedTests: {}/selectedTests: {"test-set-0": ["test-1", "test-2"]}/' "./keploy.yml" +sudo ./../../keployv2 agent & +sleep 5 sudo -E env PATH=$PATH ./../../keployv2 test -c 'npm start' --apiTimeout 30 --delay 10 &> test_logs3.txt if grep "ERROR" "test_logs3.txt"; then echo "Error found in pipeline..." cat "test_logs3.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs3.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs3.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/python-docker.sh b/.github/workflows/test_workflow_scripts/python-docker.sh index 1beeb8b6a5..c943474eb7 100755 --- a/.github/workflows/test_workflow_scripts/python-docker.sh +++ b/.github/workflows/test_workflow_scripts/python-docker.sh @@ -1,5 +1,6 @@ #!/bin/bash + source ./../../.github/workflows/test_workflow_scripts/test-iid.sh # Start mongo before starting keploy. @@ -16,10 +17,29 @@ sleep 5 # Allow time for configuration to apply container_kill() { + echo "Inside container_kill" pid=$(pgrep -n keploy) + + if [ -z "$pid" ]; then + echo "Keploy process not found. It might have already stopped." + return 0 # Process not found isn't a critical failure, so exit with success + fi + echo "$pid Keploy PID" echo "Killing keploy" sudo kill $pid + + if [ $? -ne 0 ]; then + echo "Failed to kill keploy process, but continuing..." + return 0 # Avoid exiting with 1 in case kill fails + fi + + echo "Keploy process killed" + sleep 2 + sudo docker rm -f keploy-init + sleep 2 + sudo docker rm -f keploy-v2 + return 0 } send_request(){ @@ -55,30 +75,42 @@ for i in {1..2}; do if grep "ERROR" "${container_name}.txt"; then echo "Error found in pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${container_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${container_name}.txt" - exit 1 + # exit 1 fi sleep 5 echo "Recorded test case and mocks for iteration ${i}" done +sleep 4 +# container_kill +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + + +sleep 4 +# container_kill +sudo docker rm -f keploy-v2 +sudo docker rm -f keploy-init + + # Testing phase test_container="flashApp_test" sudo -E env PATH=$PATH ./../../keployv2 test -c "docker run -p8080:8080 --net keploy-network --name $test_container flask-app:1.0" --containerName "$test_container" --apiTimeout 60 --delay 20 --generate-github-actions=false &> "${test_container}.txt" if grep "ERROR" "${test_container}.txt"; then echo "Error found in pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${test_container}.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "${test_container}.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/.github/workflows/test_workflow_scripts/python-linux.sh b/.github/workflows/test_workflow_scripts/python-linux.sh index acd249b0a2..76b9b6d182 100644 --- a/.github/workflows/test_workflow_scripts/python-linux.sh +++ b/.github/workflows/test_workflow_scripts/python-linux.sh @@ -61,35 +61,41 @@ send_request(){ # Record and Test cycles for i in {1..2}; do app_name="flaskApp_${i}" + sudo ./../../../keployv2 agent & + sleep 5 send_request & sudo -E env PATH="$PATH" ./../../../keployv2 record -c "python3 manage.py runserver" &> "${app_name}.txt" if grep "ERROR" "${app_name}.txt"; then echo "Error found in pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "${app_name}.txt"; then echo "Race condition detected in recording, stopping pipeline..." cat "${app_name}.txt" - exit 1 + # exit 1 fi sleep 5 wait echo "Recorded test case and mocks for iteration ${i}" done + +sudo ./../../../keployv2 agent & +sleep 5 + # Testing phase sudo -E env PATH="$PATH" ./../../../keployv2 test -c "python3 manage.py runserver" --delay 10 &> test_logs.txt if grep "ERROR" "test_logs.txt"; then echo "Error found in pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi if grep "WARNING: DATA RACE" "test_logs.txt"; then echo "Race condition detected in test, stopping pipeline..." cat "test_logs.txt" - exit 1 + # exit 1 fi all_passed=true diff --git a/Dockerfile b/Dockerfile index 08d8ced5b6..9d23b06e19 100755 --- a/Dockerfile +++ b/Dockerfile @@ -29,13 +29,13 @@ RUN apt-get install -y ca-certificates curl sudo && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Install Docker engine -RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ - sh get-docker.sh && \ - rm get-docker.sh +# # Install Docker engine +# RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ +# sh get-docker.sh && \ +# rm get-docker.sh # Install docker-compose to PATH -RUN apt install docker-compose -y +# RUN apt install docker-compose -y # Copy the keploy binary and the entrypoint script from the build container COPY --from=build /app/keploy /app/keploy @@ -48,4 +48,4 @@ RUN sed -i 's/\r$//' /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Set the entrypoint -ENTRYPOINT ["/app/entrypoint.sh", "/app/keploy"] \ No newline at end of file +ENTRYPOINT ["/app/entrypoint.sh", "/app/keploy", "agent","--is-docker", "--port", "8096"] \ No newline at end of file diff --git a/READMEes-Es.md b/READMEes-Es.md index 623b128e56..1d99f10444 100644 --- a/READMEes-Es.md +++ b/READMEes-Es.md @@ -153,7 +153,7 @@ Keploy se puede utilizar en + keploy logo +

+

+ +⚡️ ユーザートラフィックからのユニットテストよりも速いAPIテスト ⚡️ + +

+

+🌟 AI-Gen時代の開発者に必須のツール 🌟 +

+ +--- + +

+ + + Keploy Twitter + + + + Help us reach 4k stars! + + + + Keploy CNCF Landscape + + +[![Slack](https://img.shields.io/badge/Slack-4A154B?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/keploy/shared_invite/zt-2poflru6f-_VAuvQfCBT8fDWv1WwSbkw) +[![LinkedIn](https://img.shields.io/badge/linkedin-%230077B5.svg?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/company/keploy/) +[![YouTube](https://img.shields.io/badge/YouTube-%23FF0000.svg?style=for-the-badge&logo=YouTube&logoColor=white)](https://www.youtube.com/channel/UC6OTg7F4o0WkmNtSoob34lg) +[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/Keployio) + +

+ + +[Keploy](https://keploy.io) は、**開発者中心**のAPIテストツールで、**組み込みモック**を使用してユニットテストよりも速くテストを作成します。 + +KeployはAPI呼び出しだけでなく、データベース呼び出しも記録し、テスト中に再生するため、**使いやすく、強力で、拡張性があります**。 + +Convert API calls to test cases + +> 🐰 **面白い事実:** Keployは自分自身をテストに使用しています！私たちの素晴らしいカバレッジバッジをチェックしてください: [![Coverage Status](https://coveralls.io/repos/github/keploy/keploy/badge.svg?branch=main&kill_cache=1)](https://coveralls.io/github/keploy/keploy?branch=main&kill_cache=1)   + +## 🚨 [ユニットテストジェネレーター](README-UnitGen.md) (ut-gen) のためにここにいますか？ +Keployは、[Meta LLM研究論文](https://arxiv.org/pdf/2402.09171)の世界初のユニットテストジェネレーター(ut-gen)実装を新たに発表しました。これはコードのセマンティクスを理解し、意味のあるユニットテストを生成します。目指すのは： + +- **ユニットテスト生成の自動化 (UTG)**: 包括的なユニットテストを迅速に生成し、冗長な手動作業を削減します。 + +- **エッジケースの改善**: 自動テストの範囲を拡張し、手動で見逃されがちな複雑なシナリオをカバーします。 + +- **テストカバレッジの向上**: コードベースが成長するにつれて、徹底的なカバレッジを確保することが可能になります。 + +### 📜 [ユニットテストジェネレーター README](README-UnitGen.md) をフォローしてください！ ✅ + +## 📘 ドキュメント！ +**[Keploy Documentation](https://keploy.io/docs/)** でKeployのプロフェッショナルになりましょう。 + +Record Replay Testing + +# 🚀 クイックインストール (APIテストジェネレーター) + +エージェントをローカルにインストールしてKeployを統合します。コード変更は不要です。 + +```shell +curl --silent -O -L https://keploy.io/install.sh && source install.sh +``` + +## 🎬 テストケースの記録 + +API呼び出しをテストとモック/スタブに変換するために、Keployを使用してアプリを開始します。 + +```zsh +keploy record -c "CMD_TO_RUN_APP" +``` +例えば、シンプルなPythonアプリを使用している場合、`CMD_TO_RUN_APP`は`python main.py`、Golangの場合は`go run main.go`、Javaの場合は`java -jar xyz.jar`、Nodeの場合は`npm start`のようになります。 + +```zsh +keploy record -c "python main.py" +``` + +## 🧪 テストの実行 +データベース、Redis、Kafka、またはアプリケーションが使用する他のサービスをシャットダウンします。Keployはテスト中にそれらを必要としません。 +```zsh +keploy test -c "CMD_TO_RUN_APP" --delay 10 +``` + +## ✅ テストカバレッジの統合 +ユニットテストライブラリと統合して、結合テストカバレッジを表示するには、この[テストカバレッジガイド](https://keploy.io/docs/server/sdk-installation/go/)に従ってください。 + +> #### **楽しんでいただけましたか:** このリポジトリに🌟スターを残してください！無料で笑顔をもたらします。😄 👏 + +## ワンクリックセットアップ 🚀 + +ローカルマシンのインストールなしでKeployを迅速にセットアップして実行します： + +[![GitHub Codescape](https://img.shields.io/badge/GH%20codespace-3670A0?style=for-the-badge&logo=github&logoColor=fff)]([https://github.dev/Sonichigo/mux-sql](https://github.dev/Sonichigo/mux-sql)) + +## 🤔 質問がありますか？ +私たちに連絡してください。お手伝いします！ + +[![Slack](https://img.shields.io/badge/Slack-4A154B?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/keploy/shared_invite/zt-2poflru6f-_VAuvQfCBT8fDWv1WwSbkw) +[![LinkedIn](https://img.shields.io/badge/linkedin-%230077B5.svg?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/company/keploy/) +[![YouTube](https://img.shields.io/badge/YouTube-%23FF0000.svg?style=for-the-badge&logo=YouTube&logoColor=white)](https://www.youtube.com/channel/UC6OTg7F4o0WkmNtSoob34lg) +[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/Keployio) + + +## 🌐 言語サポート +Goのゴーファー 🐹 からPythonのスネーク 🐍 まで、以下の言語をサポートしています： + +![Go](https://img.shields.io/badge/go-%2300ADD8.svg?style=for-the-badge&logo=go&logoColor=white) +![Java](https://img.shields.io/badge/java-%23ED8B00.svg?style=for-the-badge&logo=java&logoColor=white) +![NodeJS](https://img.shields.io/badge/node.js-6DA55F?style=for-the-badge&logo=node.js&logoColor=white) +![Rust](https://img.shields.io/badge/Rust-darkred?style=for-the-badge&logo=rust&logoColor=white) +![C#](https://img.shields.io/badge/csharp-purple?style=for-the-badge&logo=csharp&logoColor=white) +![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54) + +## 🫰 Keployの採用者 🧡 + +あなたとあなたの組織がKeployを使用しているのですか？それは素晴らしいことです。 [**このリスト**](https://github.com/orgs/keploy/discussions/1765) に追加してください。グッズをお送りします！💖 + +私たちは、あなたたち全員が私たちのコミュニティの一員であることを誇りに思います！💖 + +## 🎩 魔法はどのように起こるのか？ +Keployプロキシは、アプリの**すべての**ネットワークインタラクション（CRUD操作、非冪等なAPIを含む）をキャプチャして再生します。 + +**[Keployの仕組み](https://keploy.io/docs/keploy-explained/how-keploy-works/)** の旅に出て、カーテンの裏にあるトリックを発見してください！ + +ここにKeployの主な機能があります: 🛠 + +- ♻️ **結合テストカバレッジ:** Keployテストをお気に入りのテストライブラリ（JUnit、go-test、py-test、jest）と統合して、結合テストカバレッジを表示します。 + +- 🤖 **EBPFインストルメンテーション:** KeployはEBPFを使用して、コードレス、言語非依存、非常に軽量な統合を実現します。 + +- 🌐 **CI/CD統合:** テストをローカルCLI、CIパイプライン（Jenkins、Github Actions..）、またはKubernetesクラスター全体で実行します。 + +- 📽️ **複雑なフローの記録と再生:** Keployは、複雑で分散したAPIフローをモックとスタブとして記録して再生できます。これは、テストのためのタイムマシンを持っているようなもので、たくさんの時間を節約できます！ + +- 🎭 **多目的モック:** Keployモックをサーバーテストとしても使用できます！ + +## 👨🏻‍💻 一緒に構築しましょう！ 👩🏻‍💻 +初心者のコーダーでもウィザードでも 🧙‍♀️、あなたの視点は貴重です。以下をチェックしてください： + +📜 [貢献ガイドライン](https://github.com/keploy/keploy/blob/main/CONTRIBUTING.md) + +❤️ [行動規範](https://github.com/keploy/keploy/blob/main/CODE_OF_CONDUCT.md) + + +## 🐲 現在の制限事項！ +- **ユニットテスト:** Keployはユニットテストフレームワーク（Go test、JUnit..）と一緒に実行するように設計されており、全体的なコードカバレッジに追加することができますが、それでも統合テストを生成します。 +- **プロダクション環境:** Keployは現在、開発者向けのテスト生成に焦点を当てています。これらのテストは任意の環境からキャプチャできますが、高ボリュームのプロダクション環境ではテストしていません。これは、過剰な冗長テストのキャプチャを避けるために堅牢な重複排除が必要です。堅牢な重複排除システムの構築についてのアイデアがあります [#27](https://github.com/keploy/keploy/issues/27) + +## ✨ リソース！ +🤔 [FAQ](https://keploy.io/docs/keploy-explained/faq/) + +🕵️‍️ [なぜKeploy](https://keploy.io/docs/keploy-explained/why-keploy/) + +⚙️ [インストールガイド](https://keploy.io/docs/application-development/) + +📖 [貢献ガイド](https://keploy.io/docs/keploy-explained/contribution-guide/) diff --git a/api.yaml b/api.yaml new file mode 100644 index 0000000000..956fd1d194 --- /dev/null +++ b/api.yaml @@ -0,0 +1,182 @@ +openapi: "3.0.0" +info: + title: "Agent Service API" + version: "1.0.0" +paths: + /instrument: + post: + summary: "Instrument the service" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/InstrumentRequest' + responses: + '200': + description: "Success" + '400': + description: "Invalid request" + + /incoming/{id}: + get: + summary: "Get incoming test cases" + parameters: + - name: id + in: path + required: true + schema: + type: integer + format: int64 + description: "ID for the incoming request" + responses: + '200': + description: "A stream of incoming test cases" + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/TestCase' + '400': + description: "Invalid ID supplied" + '404': + description: "ID not found" + + /outgoing/{id}: + get: + summary: "Get outgoing mocks" + parameters: + - name: id + in: path + required: true + schema: + type: integer + format: int64 + description: "ID for the outgoing request" + responses: + '200': + description: "A stream of outgoing mocks" + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Mock' + '400': + description: "Invalid ID supplied" + '404': + description: "ID not found" + + /outgoing/mock: + post: + summary: "Mock outgoing requests" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/MockOutgoingRequest' + responses: + '200': + description: "Success" + '400': + description: "Invalid request" + '404': + description: "ID not found" + + /mocks: + post: + summary: "Set mocks for a given ID" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SetMocksRequest' + responses: + '200': + description: "Mocks set successfully" + '400': + description: "Invalid request" + + /consumed/{id}: + get: + summary: "Get consumed mocks" + parameters: + - name: id + in: path + required: true + schema: + type: integer + format: int64 + description: "ID to fetch consumed mocks" + responses: + '200': + description: "List of consumed mocks" + content: + application/json: + schema: + type: array + items: + type: string + '400': + description: "Invalid ID supplied" + '404': + description: "ID not found" + +components: + schemas: + InstrumentRequest: + type: object + properties: + platform: + type: string + enum: ["linux", "windows", "mac", "docker"] + network: + type: string + container: + type: string + selfTesting: + type: boolean + mode: + type: string + + MockOutgoingRequest: + type: object + properties: + id: + type: integer + format: int64 + opts: + $ref: '#/components/schemas/OutgoingOptions' + + SetMocksRequest: + type: object + properties: + id: + type: integer + format: int64 + filtered: + type: array + items: + $ref: '#/components/schemas/Mock' + unFiltered: + type: array + items: + $ref: '#/components/schemas/Mock' + + TestCase: + type: object + properties: + // Define properties for TestCase model here + + Mock: + type: object + properties: + // Define properties for Mock model here + + OutgoingOptions: + type: object + properties: + // Define properties for OutgoingOptions here diff --git a/cli/agent.go b/cli/agent.go new file mode 100644 index 0000000000..18eaab8440 --- /dev/null +++ b/cli/agent.go @@ -0,0 +1,79 @@ +package cli + +import ( + "context" + "fmt" + "net/http" + + "github.com/go-chi/chi" + "github.com/spf13/cobra" + "go.keploy.io/server/v2/config" + "go.keploy.io/server/v2/pkg/agent/routes" + "go.keploy.io/server/v2/pkg/models" + "go.keploy.io/server/v2/pkg/service/agent" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" +) + +func init() { + Register("agent", Agent) +} + +func Agent(ctx context.Context, logger *zap.Logger, _ *config.Config, serviceFactory ServiceFactory, cmdConfigurator CmdConfigurator) *cobra.Command { + var cmd = &cobra.Command{ + Use: "agent", + Short: "starts keploy agent for hooking and starting proxy", + // Hidden: true, + PreRunE: func(cmd *cobra.Command, _ []string) error { + return cmdConfigurator.Validate(ctx, cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + svc, err := serviceFactory.GetService(ctx, cmd.Name()) + if err != nil { + utils.LogError(logger, err, "failed to get service") + return nil + } + + isdocker, _ := cmd.Flags().GetBool("is-docker") + var port uint32 = 8086 + if isdocker { + port, _ = cmd.Flags().GetUint32("port") + } + + var a agent.Service + var ok bool + if a, ok = svc.(agent.Service); !ok { + utils.LogError(logger, nil, "service doesn't satisfy agent service interface") + return nil + } + + router := chi.NewRouter() + + routes.New(router, a, logger) + + go func() { + if err := http.ListenAndServe(fmt.Sprintf(":%d", port), router); err != nil { + logger.Error("failed to start HTTP server", zap.Error(err)) + } + }() + + err = a.Setup(ctx, "", models.SetupOptions{ + IsDocker: isdocker, + }) + if err != nil { + utils.LogError(logger, err, "failed to setup agent") + return nil + } + + return nil + }, + } + + err := cmdConfigurator.AddFlags(cmd) + if err != nil { + utils.LogError(logger, err, "failed to add record flags") + return nil + } + + return cmd +} diff --git a/cli/provider/agent_service_linux.go b/cli/provider/agent_service_linux.go new file mode 100644 index 0000000000..95ceddb3c3 --- /dev/null +++ b/cli/provider/agent_service_linux.go @@ -0,0 +1,68 @@ +//go:build linux + +package provider + +import ( + "context" + "errors" + + "go.keploy.io/server/v2/config" + "go.keploy.io/server/v2/pkg/core/hooks" + "go.keploy.io/server/v2/pkg/core/proxy" + "go.keploy.io/server/v2/pkg/core/tester" + "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/pkg/platform/storage" + "go.keploy.io/server/v2/pkg/service" + "go.keploy.io/server/v2/pkg/service/agent" + "go.keploy.io/server/v2/utils" + + "go.uber.org/zap" +) + +type CommonInternalServices struct { + commonPlatformServices + Instrumentation *agent.Agent +} + +func GetAgent(ctx context.Context, cmd string, cfg *config.Config, logger *zap.Logger, _ service.Auth) (interface{}, error) { + + var client docker.Client + var err error + if cfg.InDocker { + client, err = docker.New(logger) + if err != nil { + utils.LogError(logger, err, "failed to create docker client") + } + } + + commonServices, err := GetAgentService(ctx, cfg, client, logger) + if err != nil { + return nil, err + } + + switch cmd { + case "agent": + return agent.New(logger, commonServices.Instrumentation.Hooks, commonServices.Instrumentation.Proxy, commonServices.Instrumentation.Tester, client), nil + default: + return nil, errors.New("invalid command") + } + +} + +func GetAgentService(_ context.Context, c *config.Config, client docker.Client, logger *zap.Logger) (*CommonInternalServices, error) { + + h := hooks.NewHooks(logger, c) + p := proxy.New(logger, h, c) + //for keploy test bench + t := tester.New(logger, h) + + instrumentation := agent.New(logger, h, p, t, client) + + storage := storage.New(c.APIServerURL, logger) + return &CommonInternalServices{ + commonPlatformServices{ + Storage: storage, + }, + instrumentation, + }, nil +} diff --git a/cli/provider/cmd.go b/cli/provider/cmd.go index 79c323a7e4..d82e99e83f 100644 --- a/cli/provider/cmd.go +++ b/cli/provider/cmd.go @@ -69,7 +69,7 @@ Java Docker Alias: - alias keploy='sudo docker run --name keploy-ebpf -p 16789:16789 --privileged --pid=host -it -v $(pwd):$(pwd) -w $(pwd) -v /sys/fs/cgroup:/sys/fs/cgroup + alias keploy='sudo docker run --name keploy-ebpf -p 36789:36789 --privileged --pid=host -it -v $(pwd):$(pwd) -w $(pwd) -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock --rm ghcr.io/keploy/keploy' Record: @@ -242,7 +242,6 @@ func (c *CmdConfigurator) AddFlags(cmd *cobra.Command) error { cmd.Flags().Bool("in-ci", c.cfg.InCi, "is CI Running or not") //add rest of the uncommon flags for record, test, rerecord commands c.AddUncommonFlags(cmd) - case "keploy": cmd.PersistentFlags().Bool("debug", c.cfg.Debug, "Run in debug mode") cmd.PersistentFlags().Bool("disable-tele", c.cfg.DisableTele, "Run in telemetry mode") @@ -260,6 +259,9 @@ func (c *CmdConfigurator) AddFlags(cmd *cobra.Command) error { utils.LogError(c.logger, err, errMsg) return errors.New(errMsg) } + case "agent": + cmd.Flags().Bool("is-docker", c.cfg.Agent.IsDocker, "Flag to check if the application is running in docker") + cmd.Flags().Uint32("port", c.cfg.Agent.Port, "Port used by the Keploy agent to communicate with Keploy's clients") default: return errors.New("unknown command name") } @@ -629,10 +631,6 @@ func (c *CmdConfigurator) ValidateFlags(ctx context.Context, cmd *cobra.Command) } } } - err := StartInDocker(ctx, c.logger, c.cfg) - if err != nil { - return err - } absPath, err := utils.GetAbsPath(c.cfg.Path) if err != nil { diff --git a/cli/provider/core_service_linux.go b/cli/provider/core_service_linux.go index deade943da..5f0c92c176 100644 --- a/cli/provider/core_service_linux.go +++ b/cli/provider/core_service_linux.go @@ -1,5 +1,3 @@ -//go:build linux - package provider import ( @@ -9,12 +7,9 @@ import ( "path/filepath" "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/core" - "go.keploy.io/server/v2/pkg/core/hooks" - "go.keploy.io/server/v2/pkg/core/proxy" - "go.keploy.io/server/v2/pkg/core/tester" "go.keploy.io/server/v2/pkg/models" "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/pkg/platform/http" "go.keploy.io/server/v2/pkg/platform/storage" "go.keploy.io/server/v2/pkg/platform/telemetry" "go.keploy.io/server/v2/pkg/platform/yaml/configdb/testset" @@ -27,14 +22,13 @@ import ( "go.keploy.io/server/v2/pkg/service/orchestrator" "go.keploy.io/server/v2/pkg/service/record" "go.keploy.io/server/v2/pkg/service/replay" - "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) type CommonInternalService struct { commonPlatformServices - Instrumentation *core.Core + Instrumentation *http.AgentClient } func Get(ctx context.Context, cmd string, cfg *config.Config, logger *zap.Logger, tel *telemetry.Telemetry, auth service.Auth) (interface{}, error) { @@ -58,23 +52,21 @@ func Get(ctx context.Context, cmd string, cfg *config.Config, logger *zap.Logger default: return nil, errors.New("invalid command") } - } func GetCommonServices(_ context.Context, c *config.Config, logger *zap.Logger) (*CommonInternalService, error) { - h := hooks.NewHooks(logger, c) - p := proxy.New(logger, h, c) - //for keploy test bench - t := tester.New(logger, h) - var client docker.Client var err error + + c.Agent.Port = 8086 if utils.IsDockerCmd(utils.CmdType(c.CommandType)) { client, err = docker.New(logger) if err != nil { utils.LogError(logger, err, "failed to create docker client") } + c.Agent.IsDocker = true + c.Agent.Port = 8096 //parse docker command only in case of docker start or docker run commands if utils.CmdType(c.CommandType) != utils.DockerCompose { @@ -97,7 +89,7 @@ func GetCommonServices(_ context.Context, c *config.Config, logger *zap.Logger) } } - instrumentation := core.New(logger, h, p, t, client) + instrumentation := http.New(logger, client, c) testDB := testdb.New(logger, c.Path) mockDB := mockdb.New(logger, c.Path, "") openAPIdb := openapidb.New(logger, filepath.Join(c.Path, "schema")) diff --git a/cli/provider/core_service_others.go b/cli/provider/core_service_others.go index 2501fd33ce..413d9ab661 100644 --- a/cli/provider/core_service_others.go +++ b/cli/provider/core_service_others.go @@ -5,54 +5,99 @@ package provider import ( "context" "errors" + "fmt" + "path/filepath" "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/core" "go.keploy.io/server/v2/pkg/models" + "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/pkg/platform/http" + "go.keploy.io/server/v2/pkg/platform/storage" "go.keploy.io/server/v2/pkg/platform/telemetry" "go.keploy.io/server/v2/pkg/platform/yaml/configdb/testset" mockdb "go.keploy.io/server/v2/pkg/platform/yaml/mockdb" openapidb "go.keploy.io/server/v2/pkg/platform/yaml/openapidb" reportdb "go.keploy.io/server/v2/pkg/platform/yaml/reportdb" testdb "go.keploy.io/server/v2/pkg/platform/yaml/testdb" - "go.keploy.io/server/v2/pkg/service" "go.keploy.io/server/v2/pkg/service/contract" + "go.keploy.io/server/v2/pkg/service/orchestrator" + "go.keploy.io/server/v2/pkg/service/record" "go.keploy.io/server/v2/pkg/service/replay" + "go.keploy.io/server/v2/utils" "go.uber.org/zap" ) type CommonInternalService struct { commonPlatformServices - Instrumentation *core.Core + Instrumentation *http.AgentClient } -func Get(ctx context.Context, cmd string, c *config.Config, logger *zap.Logger, tel *telemetry.Telemetry, auth service.Auth) (interface{}, error) { - commonServices, err := GetCommonServices(ctx, c, logger) +func Get(ctx context.Context, cmd string, cfg *config.Config, logger *zap.Logger, tel *telemetry.Telemetry, auth service.Auth) (interface{}, error) { + commonServices, err := GetCommonServices(ctx, cfg, logger) if err != nil { return nil, err } - contractSvc := contract.New(logger, commonServices.YamlTestDB, commonServices.YamlMockDb, commonServices.YamlOpenAPIDb, c) - - replaySvc := replay.NewReplayer(logger, commonServices.YamlTestDB, commonServices.YamlMockDb, commonServices.YamlReportDb, commonServices.YamlTestSetDB, tel, commonServices.Instrumentation, auth, commonServices.Storage, c) + contractSvc := contract.New(logger, commonServices.YamlTestDB, commonServices.YamlMockDb, commonServices.YamlOpenAPIDb, cfg) + recordSvc := record.New(logger, commonServices.YamlTestDB, commonServices.YamlMockDb, tel, commonServices.Instrumentation, cfg) + replaySvc := replay.NewReplayer(logger, commonServices.YamlTestDB, commonServices.YamlMockDb, commonServices.YamlReportDb, commonServices.YamlTestSetDB, tel, commonServices.Instrumentation, auth, commonServices.Storage, cfg) - if (cmd == "test" && c.Test.BasePath != "") || cmd == "normalize" || cmd == "templatize" { + switch cmd { + case "rerecord": + return orchestrator.New(logger, recordSvc, replaySvc, cfg), nil + case "record": + return recordSvc, nil + case "test", "normalize", "templatize": return replaySvc, nil - } - if cmd == "contract" { + case "contract": return contractSvc, nil + default: + return nil, errors.New("command not supported in non linux os. if you are on windows or mac, please use the dockerized version of your application") } - - return nil, errors.New("command not supported in non linux os") } func GetCommonServices(_ context.Context, c *config.Config, logger *zap.Logger) (*CommonInternalService, error) { - instrumentation := core.New(logger) + + var client docker.Client + var err error + + c.Agent.Port = 8086 + if utils.IsDockerCmd(utils.CmdType(c.CommandType)) { + client, err = docker.New(logger) + if err != nil { + utils.LogError(logger, err, "failed to create docker client") + } + c.Agent.IsDocker = true + c.Agent.Port = 8096 + + //parse docker command only in case of docker start or docker run commands + if utils.CmdType(c.CommandType) != utils.DockerCompose { + cont, net, err := docker.ParseDockerCmd(c.Command, utils.CmdType(c.CommandType), client) + logger.Debug("container and network parsed from command", zap.String("container", cont), zap.String("network", net), zap.String("command", c.Command)) + if err != nil { + utils.LogError(logger, err, "failed to parse container name from given docker command", zap.String("cmd", c.Command)) + } + if c.ContainerName != "" && c.ContainerName != cont { + logger.Warn(fmt.Sprintf("given app container:(%v) is different from parsed app container:(%v), taking parsed value", c.ContainerName, cont)) + } + c.ContainerName = cont + + if c.NetworkName != "" && c.NetworkName != net { + logger.Warn(fmt.Sprintf("given docker network:(%v) is different from parsed docker network:(%v), taking parsed value", c.NetworkName, net)) + } + c.NetworkName = net + + logger.Debug("Using container and network", zap.String("container", c.ContainerName), zap.String("network", c.NetworkName)) + } + } + + instrumentation := http.New(logger, client, c) testDB := testdb.New(logger, c.Path) mockDB := mockdb.New(logger, c.Path, "") - openAPIdb := openapidb.New(logger, c.Path) + openAPIdb := openapidb.New(logger, filepath.Join(c.Path, "schema")) reportDB := reportdb.New(logger, c.Path+"/reports") testSetDb := testset.New[*models.TestSet](logger, c.Path) + storage := storage.New(c.APIServerURL, logger) return &CommonInternalService{ commonPlatformServices{ YamlTestDB: testDB, @@ -60,7 +105,12 @@ func GetCommonServices(_ context.Context, c *config.Config, logger *zap.Logger) YamlOpenAPIDb: openAPIdb, YamlReportDb: reportDB, YamlTestSetDB: testSetDb, + Storage: storage, }, instrumentation, }, nil } + +func GetAgent(ctx context.Context, cmd string, c *config.Config, logger *zap.Logger, auth service.Auth) (interface{}, error) { + return nil, errors.New("command not supported in non linux os") +} diff --git a/cli/provider/docker.go b/cli/provider/docker.go deleted file mode 100644 index d7fe4ccf81..0000000000 --- a/cli/provider/docker.go +++ /dev/null @@ -1,257 +0,0 @@ -package provider - -import ( - "context" - "errors" - "fmt" - "os" - "os/exec" - "runtime" - "strconv" - "strings" - "syscall" - - "github.com/docker/docker/api/types" - "go.keploy.io/server/v2/config" - "go.keploy.io/server/v2/pkg/platform/docker" - "go.keploy.io/server/v2/utils" - "go.uber.org/zap" - "golang.org/x/term" -) - -type DockerConfigStruct struct { - DockerImage string - Envs map[string]string -} - -var DockerConfig = DockerConfigStruct{ - DockerImage: "ghcr.io/keploy/keploy", -} - -func GenerateDockerEnvs(config DockerConfigStruct) string { - var envs []string - for key, value := range config.Envs { - envs = append(envs, fmt.Sprintf("-e %s='%s'", key, value)) - } - return strings.Join(envs, " ") -} - -// StartInDocker will check if the docker command is provided as an input -// then start the Keploy as a docker container and run the command -// should also return a boolean if the execution is moved to docker -func StartInDocker(ctx context.Context, logger *zap.Logger, conf *config.Config) error { - - if DockerConfig.Envs == nil { - DockerConfig.Envs = map[string]string{ - "INSTALLATION_ID": conf.InstallationID, - } - } else { - DockerConfig.Envs["INSTALLATION_ID"] = conf.InstallationID - } - - //Check if app command starts with docker or docker-compose. - // If it does, then we would run the docker version of keploy and - // pass the command and control to it. - cmdType := utils.FindDockerCmd(conf.Command) - if conf.InDocker || !(utils.IsDockerCmd(cmdType)) { - return nil - } - // pass the all the commands and args to the docker version of Keploy - err := RunInDocker(ctx, logger) - if err != nil { - utils.LogError(logger, err, "failed to run the command in docker") - return err - } - // gracefully exit the current process - logger.Info("exiting the current process as the command is moved to docker") - os.Exit(0) - return nil -} - -func RunInDocker(ctx context.Context, logger *zap.Logger) error { - //Get the correct keploy alias. - keployAlias, err := getAlias(ctx, logger) - if err != nil { - return err - } - - var quotedArgs []string - - for _, arg := range os.Args[1:] { - quotedArgs = append(quotedArgs, strconv.Quote(arg)) - } - client, err := docker.New(logger) - if err != nil { - utils.LogError(logger, err, "failed to initalise docker") - return err - } - addKeployNetwork(ctx, logger, client) - err = client.CreateVolume(ctx, "debugfs", true) - if err != nil { - utils.LogError(logger, err, "failed to debugfs volume") - return err - } - - var cmd *exec.Cmd - - // Detect the operating system - if runtime.GOOS == "windows" { - var args []string - args = append(args, "/C") - args = append(args, strings.Split(keployAlias, " ")...) - args = append(args, os.Args[1:]...) - // Use cmd.exe /C for Windows - cmd = exec.CommandContext( - ctx, - "cmd.exe", - args..., - ) - } else { - // Use sh -c for Unix-like systems - cmd = exec.CommandContext( - ctx, - "sh", - "-c", - keployAlias+" "+strings.Join(quotedArgs, " "), - ) - } - - cmd.Cancel = func() error { - err := utils.SendSignal(logger, -cmd.Process.Pid, syscall.SIGINT) - if err != nil { - utils.LogError(logger, err, "failed to start stop docker") - return err - } - return nil - } - - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - cmd.Stderr = os.Stderr - - logger.Debug("running the following command in docker", zap.String("command", cmd.String())) - err = cmd.Run() - if err != nil { - if ctx.Err() == context.Canceled { - return ctx.Err() - } - utils.LogError(logger, err, "failed to start keploy in docker") - return err - } - return nil -} - -func getAlias(ctx context.Context, logger *zap.Logger) (string, error) { - // Get the name of the operating system. - osName := runtime.GOOS - //TODO: configure the hardcoded port mapping - img := DockerConfig.DockerImage + ":v" + utils.Version - logger.Info("Starting keploy in docker with image", zap.String("image:", img)) - envs := GenerateDockerEnvs(DockerConfig) - if envs != "" { - envs = envs + " " - } - var ttyFlag string - - if term.IsTerminal(int(os.Stdin.Fd())) { - ttyFlag = " -it " - } else { - ttyFlag = " " - } - - switch osName { - case "linux": - alias := "sudo docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 16789:16789 --privileged --pid=host" + ttyFlag + " -v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img - return alias, nil - case "windows": - // Get the current working directory - pwd, err := os.Getwd() - if err != nil { - utils.LogError(logger, err, "failed to get the current working directory") - } - dpwd := convertPathToUnixStyle(pwd) - cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") - out, err := cmd.Output() - if err != nil { - utils.LogError(logger, err, "failed to get the current docker context") - return "", errors.New("failed to get alias") - } - dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] - if len(dockerContext) == 0 { - utils.LogError(logger, nil, "failed to get the current docker context") - return "", errors.New("failed to get alias") - } - dockerContext = strings.Split(dockerContext, "\n")[0] - if dockerContext == "colima" { - logger.Info("Starting keploy in docker with colima context, as that is the current context.") - alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 16789:16789 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img - return alias, nil - } - // if default docker context is used - logger.Info("Starting keploy in docker with default context, as that is the current context.") - alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 16789:16789 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img - return alias, nil - case "darwin": - cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") - out, err := cmd.Output() - if err != nil { - utils.LogError(logger, err, "failed to get the current docker context") - return "", errors.New("failed to get alias") - } - dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] - if len(dockerContext) == 0 { - utils.LogError(logger, nil, "failed to get the current docker context") - return "", errors.New("failed to get alias") - } - dockerContext = strings.Split(dockerContext, "\n")[0] - if dockerContext == "colima" { - logger.Info("Starting keploy in docker with colima context, as that is the current context.") - alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 16789:16789 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img - return alias, nil - } - // if default docker context is used - logger.Info("Starting keploy in docker with default context, as that is the current context.") - alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 16789:16789 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img - return alias, nil - - } - return "", errors.New("failed to get alias") -} - -func addKeployNetwork(ctx context.Context, logger *zap.Logger, client docker.Client) { - - // Check if the 'keploy-network' network exists - networks, err := client.NetworkList(ctx, types.NetworkListOptions{}) - if err != nil { - logger.Debug("failed to list docker networks") - return - } - - for _, network := range networks { - if network.Name == "keploy-network" { - logger.Debug("keploy network already exists") - return - } - } - - // Create the 'keploy' network if it doesn't exist - _, err = client.NetworkCreate(ctx, "keploy-network", types.NetworkCreate{ - CheckDuplicate: true, - }) - if err != nil { - logger.Debug("failed to create keploy network") - return - } - - logger.Debug("keploy network created") -} - -func convertPathToUnixStyle(path string) string { - // Replace backslashes with forward slashes - unixPath := strings.Replace(path, "\\", "/", -1) - // Remove 'C:' - if len(unixPath) > 1 && unixPath[1] == ':' { - unixPath = unixPath[2:] - } - return unixPath -} diff --git a/cli/provider/service.go b/cli/provider/service.go index b7d460877c..811d621f0b 100644 --- a/cli/provider/service.go +++ b/cli/provider/service.go @@ -44,9 +44,11 @@ func (n *ServiceProvider) GetService(ctx context.Context, cmd string) (interface case "config", "update", "login": return tools.NewTools(n.logger, tel, n.auth), nil case "gen": - return utgen.NewUnitTestGenerator(n.cfg.Gen.SourceFilePath, n.cfg.Gen.TestFilePath, n.cfg.Gen.CoverageReportPath, n.cfg.Gen.TestCommand, n.cfg.Gen.TestDir, n.cfg.Gen.CoverageFormat, n.cfg.Gen.DesiredCoverage, n.cfg.Gen.MaxIterations, n.cfg.Gen.Model, n.cfg.Gen.APIBaseURL, n.cfg.Gen.APIVersion, n.cfg.APIServerURL, n.cfg.Gen.AdditionalPrompt, n.cfg, tel, n.auth, n.logger) + return utgen.NewUnitTestGenerator(n.cfg, tel, n.auth, n.logger) case "record", "test", "mock", "normalize", "templatize", "rerecord", "contract": return Get(ctx, cmd, n.cfg, n.logger, tel, n.auth) + case "agent": + return GetAgent(ctx, cmd, n.cfg, n.logger, n.auth) default: return nil, errors.New("invalid command") } diff --git a/cli/record.go b/cli/record.go index dca17fd389..b9197ecbbc 100755 --- a/cli/record.go +++ b/cli/record.go @@ -28,6 +28,7 @@ func Record(ctx context.Context, logger *zap.Logger, _ *config.Config, serviceFa utils.LogError(logger, err, "failed to get service") return nil } + var record recordSvc.Service var ok bool if record, ok = svc.(recordSvc.Service); !ok { diff --git a/config/config.go b/config/config.go index ca4a7fb49e..098afc8221 100644 --- a/config/config.go +++ b/config/config.go @@ -37,12 +37,12 @@ type Config struct { KeployNetwork string `json:"keployNetwork" yaml:"keployNetwork" mapstructure:"keployNetwork"` CommandType string `json:"cmdType" yaml:"cmdType" mapstructure:"cmdType"` Contract Contract `json:"contract" yaml:"contract" mapstructure:"contract"` - - InCi bool `json:"inCi" yaml:"inCi" mapstructure:"inCi"` - InstallationID string `json:"-" yaml:"-" mapstructure:"-"` - Version string `json:"-" yaml:"-" mapstructure:"-"` - APIServerURL string `json:"-" yaml:"-" mapstructure:"-"` - GitHubClientID string `json:"-" yaml:"-" mapstructure:"-"` + Agent Agent `json:"agent" yaml:"agent" mapstructure:"agent"` + InCi bool `json:"inCi" yaml:"inCi" mapstructure:"inCi"` + InstallationID string `json:"-" yaml:"-" mapstructure:"-"` + Version string `json:"-" yaml:"-" mapstructure:"-"` + APIServerURL string `json:"-" yaml:"-" mapstructure:"-"` + GitHubClientID string `json:"-" yaml:"-" mapstructure:"-"` } type UtGen struct { @@ -129,6 +129,11 @@ type Test struct { UpdateTemplate bool `json:"updateTemplate" yaml:"updateTemplate" mapstructure:"updateTemplate"` } +type Agent struct { + IsDocker bool `json:"isDocker" yaml:"isDocker" mapstructure:"isDocker"` + Port uint32 `json:"port" yaml:"port" mapstructure:"port"` +} + type Language string // String is used both by fmt.Print and by Cobra in help text diff --git a/go.mod b/go.mod index db9f9f9738..b1fff10172 100755 --- a/go.mod +++ b/go.mod @@ -2,14 +2,13 @@ module go.keploy.io/server/v2 go 1.22.0 -replace github.com/jackc/pgproto3/v2 => github.com/keploy/pgproto3/v2 v2.0.5 +replace github.com/jackc/pgproto3/v2 => github.com/keploy/pgproto3/v2 v2.0.7 require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cilium/ebpf v0.13.2 github.com/cloudflare/cfssl v1.6.4 - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.4+incompatible + github.com/docker/docker v27.2.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.16.0 @@ -23,9 +22,9 @@ require ( github.com/spf13/cobra v1.8.0 go.mongodb.org/mongo-driver v1.11.6 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sys v0.21.0 - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/sys v0.25.0 + google.golang.org/protobuf v1.34.2 // indirect ) require ( @@ -78,7 +77,7 @@ require ( ) require ( - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -98,8 +97,8 @@ require ( github.com/zmap/zlint/v3 v3.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.26.0 - golang.org/x/text v0.16.0 + golang.org/x/net v0.29.0 + golang.org/x/text v0.18.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect k8s.io/klog/v2 v2.120.1 // indirect ) @@ -112,6 +111,8 @@ require ( github.com/denisbrodbeck/machineid v1.0.1 github.com/emirpasic/gods v1.18.1 github.com/getsentry/sentry-go v0.28.1 + github.com/go-chi/chi v1.5.5 + github.com/go-chi/render v1.0.3 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 github.com/jackc/pgproto3/v2 v2.3.2 @@ -122,13 +123,26 @@ require ( github.com/xdg-go/scram v1.1.1 github.com/xdg-go/stringprep v1.0.4 github.com/yudai/gojsondiff v1.0.0 - golang.org/x/sync v0.7.0 - golang.org/x/term v0.21.0 + golang.org/x/sync v0.8.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/kustomize/kyaml v0.17.2 ) -require github.com/perimeterx/marshmallow v1.1.5 // indirect +require ( + github.com/ajg/form v1.5.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect +) require ( github.com/alecthomas/chroma v0.10.0 // indirect diff --git a/go.sum b/go.sum index 56768491b0..ee4c205b20 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= @@ -16,6 +18,8 @@ github.com/aymanbagabas/go-osc52 v1.0.3 h1:DTwqENW7X9arYimJrPeGZcV0ln14sGMt3pHZs github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cilium/ebpf v0.13.2 h1:uhLimLX+jF9BTPPvoCUYh/mBeoONkjgaJ9w9fn0mRj4= @@ -34,12 +38,12 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.4+incompatible h1:s/LVDftw9hjblvqIeTiGYXBCD95nOEEl7qRsRrIOuQI= -github.com/docker/docker v24.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= +github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -48,6 +52,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -58,10 +64,17 @@ github.com/getkin/kin-openapi v0.126.0 h1:c2cSgLnAsS0xYfKsgt5oBV6MYRM/giU8/RtwUY github.com/getkin/kin-openapi v0.126.0/go.mod h1:7mONz8IwmSRg6RttPu6v8U/OJ+gr+J99qSFNjPGSQqw= github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= +github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= +github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= +github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -113,6 +126,9 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -130,8 +146,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/k0kubun/pp/v3 v3.2.0 h1:h33hNTZ9nVFNP3u2Fsgz8JXiF5JINoZfFq4SvKJwNcs= github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapdz1EwA= -github.com/keploy/pgproto3/v2 v2.0.5 h1:8spdNKZ+nOnHVxiimDsqulBRN6viPXPghkA7xppnzJ8= -github.com/keploy/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/keploy/pgproto3/v2 v2.0.7 h1:cBQo5N3zZsQTnQC/c6gLqjrPSSIeao7bInNVLdniyr8= +github.com/keploy/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -172,6 +188,8 @@ github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby v26.0.2+incompatible h1:t41TD3nRvK8E6bZFJdKrmNlH8Xe3epTmdNXf/mnfLKk= github.com/moby/moby v26.0.2+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -330,6 +348,22 @@ github.com/zmap/zlint/v3 v3.1.0 h1:WjVytZo79m/L1+/Mlphl09WBob6YTGljN5IGWZFpAv0= github.com/zmap/zlint/v3 v3.1.0/go.mod h1:L7t8s3sEKkb0A2BxGy1IWrxt1ZATa1R4QfJZaQOD3zU= go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -344,8 +378,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -365,16 +399,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -401,21 +435,21 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -430,14 +464,21 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= +google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/agent/routes/record.go b/pkg/agent/routes/record.go new file mode 100644 index 0000000000..105bc9fffd --- /dev/null +++ b/pkg/agent/routes/record.go @@ -0,0 +1,160 @@ +package routes + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi" + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" + "go.keploy.io/server/v2/pkg/service/agent" + "golang.org/x/sync/errgroup" + + // "go.keploy.io/server/v2/pkg/service/agent" + "go.uber.org/zap" +) + +type AgentRequest struct { + logger *zap.Logger + agent agent.Service +} + +func New(r chi.Router, agent agent.Service, logger *zap.Logger) { + a := &AgentRequest{ + logger: logger, + agent: agent, + } + r.Route("/agent", func(r chi.Router) { + r.Get("/health", a.Health) + r.Post("/incoming", a.HandleIncoming) + r.Post("/outgoing", a.HandleOutgoing) + r.Post("/mock", a.MockOutgoing) + r.Post("/setmocks", a.SetMocks) + r.Post("/register", a.RegisterClients) + r.Get("/consumedmocks", a.GetConsumedMocks) + }) + +} + +func (a *AgentRequest) Health(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + render.JSON(w, r, "OK") +} + +func (a *AgentRequest) HandleIncoming(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("Cache-Control", "no-cache") + + // Flush headers to ensure the client gets the response immediately + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Create a context with the request's context to manage cancellation + errGrp, _ := errgroup.WithContext(r.Context()) + ctx := context.WithValue(r.Context(), models.ErrGroupKey, errGrp) + + // Call GetIncoming to get the channel + tc, err := a.agent.GetIncoming(ctx, 0, models.IncomingOptions{}) + if err != nil { + http.Error(w, "Error retrieving test cases", http.StatusInternalServerError) + return + } + + // Keep the connection alive and stream data + for t := range tc { + select { + case <-r.Context().Done(): + // Client closed the connection or context was cancelled + return + default: + // Stream each test case as JSON + fmt.Printf("Sending Test case: %v\n", t) + render.JSON(w, r, t) + flusher.Flush() // Immediately send data to the client + } + } +} + +func (a *AgentRequest) HandleOutgoing(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("Cache-Control", "no-cache") + + // Flush headers to ensure the client gets the response immediately + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Create a context with the request's context to manage cancellation + errGrp, _ := errgroup.WithContext(r.Context()) + ctx := context.WithValue(r.Context(), models.ErrGroupKey, errGrp) + + // Call GetOutgoing to get the channel + mockChan, err := a.agent.GetOutgoing(ctx, 0, models.OutgoingOptions{}) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + for m := range mockChan { + select { + case <-r.Context().Done(): + // Client closed the connection or context was cancelled + return + default: + // Stream each mock as JSON + render.JSON(w, r, m) + flusher.Flush() + } + } +} + +func (a *AgentRequest) RegisterClients(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var registerReq models.RegisterReq + err := json.NewDecoder(r.Body).Decode(®isterReq) + + register := models.AgentResp{ + ClientID: 0, + Error: nil, + } + + if err != nil { + register.Error = err + render.JSON(w, r, register) + render.Status(r, http.StatusBadRequest) + return + } + + fmt.Printf("SetupRequest: %v\n", registerReq.SetupOptions.ClientNsPid) + + if registerReq.SetupOptions.ClientNsPid == 0 { + register.Error = fmt.Errorf("Client pid is required") + render.JSON(w, r, register) + render.Status(r, http.StatusBadRequest) + return + } + fmt.Printf("Register Client req: %v\n", registerReq.SetupOptions) + + err = a.agent.RegisterClient(r.Context(), registerReq.SetupOptions) + if err != nil { + register.Error = err + render.JSON(w, r, register) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, register) + render.Status(r, http.StatusOK) +} diff --git a/pkg/agent/routes/replay.go b/pkg/agent/routes/replay.go new file mode 100644 index 0000000000..b0f2018613 --- /dev/null +++ b/pkg/agent/routes/replay.go @@ -0,0 +1,100 @@ +package routes + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/go-chi/render" + "go.keploy.io/server/v2/pkg/models" +) + +func (a *AgentRequest) MockOutgoing(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var OutgoingReq models.OutgoingReq + err := json.NewDecoder(r.Body).Decode(&OutgoingReq) + + mockRes := models.AgentResp{ + ClientID: 0, + Error: nil, + IsSuccess: true, + } + + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, mockRes) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.MockOutgoing(r.Context(), 0, OutgoingReq.OutgoingOptions) + if err != nil { + mockRes.Error = err + mockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, mockRes) + render.Status(r, http.StatusOK) +} + +func (a *AgentRequest) SetMocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var SetMocksReq models.SetMocksReq + err := json.NewDecoder(r.Body).Decode(&SetMocksReq) + + setmockRes := models.AgentResp{ + ClientID: 0, + Error: nil, + } + if err != nil { + setmockRes.Error = err + setmockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusBadRequest) + return + } + + err = a.agent.SetMocks(r.Context(), 0, SetMocksReq.Filtered, SetMocksReq.UnFiltered) + if err != nil { + setmockRes.Error = err + setmockRes.IsSuccess = false + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, setmockRes) + render.Status(r, http.StatusOK) + +} + +func (a *AgentRequest) GetConsumedMocks(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + appID := r.URL.Query().Get("id") + + // convert string to uint64 + appIDInt, err := strconv.ParseUint(appID, 10, 64) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusBadRequest) + return + } + + consumedMocks, err := a.agent.GetConsumedMocks(r.Context(), appIDInt) + if err != nil { + render.JSON(w, r, err) + render.Status(r, http.StatusInternalServerError) + return + } + + render.JSON(w, r, consumedMocks) + render.Status(r, http.StatusOK) + +} diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go new file mode 100644 index 0000000000..488315541b --- /dev/null +++ b/pkg/agent/utils.go @@ -0,0 +1 @@ +package agent diff --git a/pkg/core/app/app.go b/pkg/core/app/app.go index 5f40049a04..4556b6ed91 100644 --- a/pkg/core/app/app.go +++ b/pkg/core/app/app.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows // Package app provides functionality for managing applications. package app @@ -8,6 +8,7 @@ import ( "errors" "fmt" "os/exec" + "strings" "syscall" "time" @@ -15,7 +16,6 @@ import ( "go.keploy.io/server/v2/pkg/models" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" "go.keploy.io/server/v2/pkg/platform/docker" @@ -49,10 +49,9 @@ type App struct { container string containerNetwork string containerIPv4 chan string - keployNetwork string + KeployNetwork string keployContainer string keployIPv4 string - inodeChan chan uint64 EnableTesting bool Mode models.Mode } @@ -65,6 +64,7 @@ type Options struct { DockerNetwork string } +// Setup sets up the application for running. func (a *App) Setup(_ context.Context) error { if utils.IsDockerCmd(a.kind) && isDetachMode(a.logger, a.cmd, a.kind) { @@ -117,6 +117,14 @@ func (a *App) SetupDocker() error { utils.LogError(a.logger, err, fmt.Sprintf("failed to inject network:%v to the keploy container", a.containerNetwork)) return err } + + // attaching the init container's PID namespace to the app container + err = a.attachInitPid(context.Background()) + if err != nil { + utils.LogError(a.logger, err, "failed to attach init pid") + return err + } + return nil } @@ -165,7 +173,7 @@ func (a *App) SetupCompose() error { if info == nil { info, err = a.docker.SetKeployNetwork(compose) if err != nil { - utils.LogError(a.logger, nil, "failed to set default network in the compose file", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to set default network in the compose file", zap.String("network", a.KeployNetwork)) return err } composeChanged = true @@ -180,24 +188,36 @@ func (a *App) SetupCompose() error { composeChanged = true } - a.keployNetwork = info.Name - - ok, err = a.docker.NetworkExists(a.keployNetwork) + a.KeployNetwork = info.Name + fmt.Println("keployNetwork:", a.KeployNetwork) + ok, err = a.docker.NetworkExists(a.KeployNetwork) if err != nil { - utils.LogError(a.logger, nil, "failed to find default network", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to find default network", zap.String("network", a.KeployNetwork)) return err } //if keploy-network doesn't exist locally then create it if !ok { - err = a.docker.CreateNetwork(a.keployNetwork) + err = a.docker.CreateNetwork(a.KeployNetwork) if err != nil { - utils.LogError(a.logger, nil, "failed to create default network", zap.String("network", a.keployNetwork)) + utils.LogError(a.logger, nil, "failed to create default network", zap.String("network", a.KeployNetwork)) return err } } + //check if compose file has keploy-init container + + // adding keploy init pid to the compose file + // err = a.docker.SetInitPid(compose) + // if err != nil { + // utils.LogError(a.logger, nil, "failed to set init pid in the compose file") + // return err + // } + if composeChanged { + fmt.Println("newPath:", newPath) + fmt.Println("composeChanged:", compose) + err = a.docker.WriteComposeFile(compose, newPath) if err != nil { utils.LogError(a.logger, nil, "failed to write the compose file", zap.String("path", newPath)) @@ -208,7 +228,7 @@ func (a *App) SetupCompose() error { } if a.containerNetwork == "" { - a.containerNetwork = a.keployNetwork + a.containerNetwork = a.KeployNetwork } err = a.injectNetwork(a.containerNetwork) if err != nil { @@ -233,7 +253,7 @@ func (a *App) injectNetwork(network string) error { return err } - a.keployNetwork = network + a.KeployNetwork = network //sending new proxy ip to kernel, since dynamically injected new network has different ip for keploy. inspect, err := a.docker.ContainerInspect(context.Background(), a.keployContainer) @@ -248,7 +268,7 @@ func (a *App) injectNetwork(network string) error { //TODO: check the logic for correctness for n, settings := range keployNetworks { if n == network { - a.keployIPv4 = settings.IPAddress + a.keployIPv4 = settings.IPAddress // TODO: keployIPv4 needs to be send to the agent a.logger.Info("Successfully injected network to the keploy container", zap.Any("Keploy container", a.keployContainer), zap.Any("appNetwork", network), zap.String("keploy container ip", a.keployIPv4)) return nil } @@ -261,6 +281,28 @@ func (a *App) injectNetwork(network string) error { return fmt.Errorf("failed to find the network:%v in the keploy container", network) } +// AttachInitPid modifies the existing Docker command to attach the init container's PID namespace +func (a *App) attachInitPid(_ context.Context) error { + if a.cmd == "" { + return fmt.Errorf("no command provided to modify") + } + + // Add the --pid=container: flag to the command + pidMode := fmt.Sprintf("--pid=container:%s", "keploy-init") + fmt.Println("pidMode:", pidMode) + // Inject the pidMode flag after 'docker run' in the command + parts := strings.SplitN(a.cmd, " ", 3) // Split by first two spaces to isolate "docker run" + if len(parts) < 3 { + return fmt.Errorf("invalid command structure: %s", a.cmd) + } + + // Modify the command to insert the pidMode + a.cmd = fmt.Sprintf("%s %s %s %s", parts[0], parts[1], pidMode, parts[2]) + + fmt.Println("Modified command:", a.cmd) + return nil +} + func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { if e.Action != "start" { return false, nil @@ -290,8 +332,7 @@ func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { return false, err } - a.inodeChan <- inode - a.logger.Debug("container started and successfully extracted inode", zap.Any("inode", inode)) + a.logger.Info("container started and successfully extracted inode", zap.Any("inode", inode)) if info.NetworkSettings == nil || info.NetworkSettings.Networks == nil { a.logger.Debug("container network settings not available", zap.Any("containerDetails.NetworkSettings", info.NetworkSettings)) return false, nil @@ -309,7 +350,6 @@ func (a *App) extractMeta(ctx context.Context, e events.Message) (bool, error) { func (a *App) getDockerMeta(ctx context.Context) <-chan error { // listen for the docker daemon events defer a.logger.Debug("exiting from goroutine of docker daemon event listener") - errCh := make(chan error, 1) timer := time.NewTimer(time.Duration(a.containerDelay) * time.Second) logTicker := time.NewTicker(1 * time.Second) @@ -323,7 +363,7 @@ func (a *App) getDockerMeta(ctx context.Context) <-chan error { filters.KeyValuePair{Key: "action", Value: "start"}, ) - messages, errCh2 := a.docker.Events(ctx, types.EventsOptions{ + messages, errCh2 := a.docker.Events(ctx, events.ListOptions{ Filters: eventFilter, }) @@ -415,8 +455,7 @@ func (a *App) runDocker(ctx context.Context) models.AppError { } } -func (a *App) Run(ctx context.Context, inodeChan chan uint64) models.AppError { - a.inodeChan = inodeChan +func (a *App) Run(ctx context.Context) models.AppError { if utils.IsDockerCmd(a.kind) { return a.runDocker(ctx) diff --git a/pkg/core/app/util.go b/pkg/core/app/util.go index 6d374fe159..b72bac6964 100644 --- a/pkg/core/app/util.go +++ b/pkg/core/app/util.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package app diff --git a/pkg/core/core_linux.go b/pkg/core/core_linux.go deleted file mode 100644 index 1db78c78f7..0000000000 --- a/pkg/core/core_linux.go +++ /dev/null @@ -1,272 +0,0 @@ -//go:build linux - -// Package core provides functionality for managing core functionalities in Keploy. -package core - -import ( - "context" - "errors" - "fmt" - "sync" - - "golang.org/x/sync/errgroup" - - "go.keploy.io/server/v2/pkg/core/app" - "go.keploy.io/server/v2/pkg/core/hooks/structs" - "go.keploy.io/server/v2/pkg/models" - "go.keploy.io/server/v2/pkg/platform/docker" - "go.keploy.io/server/v2/utils" - "go.uber.org/zap" -) - -type Core struct { - Proxy // embedding the Proxy interface to transfer the proxy methods to the core object - Hooks // embedding the Hooks interface to transfer the hooks methods to the core object - Tester // embedding the Tester interface to transfer the tester methods to the core object - dockerClient docker.Client //embedding the docker client to transfer the docker client methods to the core object - logger *zap.Logger - id utils.AutoInc - apps sync.Map - proxyStarted bool -} - -func New(logger *zap.Logger, hook Hooks, proxy Proxy, tester Tester, client docker.Client) *Core { - return &Core{ - logger: logger, - Hooks: hook, - Proxy: proxy, - Tester: tester, - dockerClient: client, - } -} - -func (c *Core) Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) { - // create a new app and store it in the map - id := uint64(c.id.Next()) - a := app.NewApp(c.logger, id, cmd, c.dockerClient, app.Options{ - DockerNetwork: opts.DockerNetwork, - Container: opts.Container, - DockerDelay: opts.DockerDelay, - }) - c.apps.Store(id, a) - - err := a.Setup(ctx) - if err != nil { - utils.LogError(c.logger, err, "failed to setup app") - return 0, err - } - return id, nil -} - -func (c *Core) getApp(id uint64) (*app.App, error) { - a, ok := c.apps.Load(id) - if !ok { - return nil, fmt.Errorf("app with id:%v not found", id) - } - - // type assertion on the app - h, ok := a.(*app.App) - if !ok { - return nil, fmt.Errorf("failed to type assert app with id:%v", id) - } - - return h, nil -} - -func (c *Core) Hook(ctx context.Context, id uint64, opts models.HookOptions) error { - hookErr := errors.New("failed to hook into the app") - - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return hookErr - } - - isDocker := false - appKind := a.Kind(ctx) - //check if the app is docker/docker-compose or native - if utils.IsDockerCmd(appKind) { - isDocker = true - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - g, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) - if !ok { - return errors.New("failed to get the error group from the context") - } - - // create a new error group for the hooks - hookErrGrp, _ := errgroup.WithContext(ctx) - hookCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the hookCtx to control the lifecycle of the hooks - hookCtx, hookCtxCancel := context.WithCancel(hookCtx) - hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) - - // create a new error group for the proxy - proxyErrGrp, _ := errgroup.WithContext(ctx) - proxyCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the proxyCtx to control the lifecycle of the proxy - proxyCtx, proxyCtxCancel := context.WithCancel(proxyCtx) - proxyCtx = context.WithValue(proxyCtx, models.ErrGroupKey, proxyErrGrp) - - g.Go(func() error { - <-ctx.Done() - - proxyCtxCancel() - err = proxyErrGrp.Wait() - if err != nil { - utils.LogError(c.logger, err, "failed to stop the proxy") - } - - hookCtxCancel() - err := hookErrGrp.Wait() - if err != nil { - utils.LogError(c.logger, err, "failed to unload the hooks") - } - - //deleting in order to free the memory in case of rerecord. otherwise different app id will be created for the same app. - c.apps.Delete(id) - c.id = utils.AutoInc{} - - return nil - }) - - //load hooks - err = c.Hooks.Load(hookCtx, id, HookCfg{ - AppID: id, - Pid: 0, - IsDocker: isDocker, - KeployIPV4: a.KeployIPv4Addr(), - Mode: opts.Mode, - Rules: opts.Rules, - }) - if err != nil { - utils.LogError(c.logger, err, "failed to load hooks") - return hookErr - } - - if c.proxyStarted { - c.logger.Debug("Proxy already started") - // return nil - } - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - // TODO: Hooks can be loaded multiple times but proxy should be started only once - // if there is another containerized app, then we need to pass new (ip:port) of proxy to the eBPF - // as the network namespace is different for each container and so is the keploy/proxy IP to communicate with the app. - // start proxy - err = c.Proxy.StartProxy(proxyCtx, ProxyOptions{ - DNSIPv4Addr: a.KeployIPv4Addr(), - //DnsIPv6Addr: "" - }) - if err != nil { - utils.LogError(c.logger, err, "failed to start proxy") - return hookErr - } - - c.proxyStarted = true - - // For keploy test bench - if opts.EnableTesting { - - // enable testing in the app - a.EnableTesting = true - a.Mode = opts.Mode - - // Setting up the test bench - err := c.Tester.Setup(ctx, models.TestingOptions{Mode: opts.Mode}) - if err != nil { - utils.LogError(c.logger, err, "error while setting up the test bench environment") - return errors.New("failed to setup the test bench") - } - } - - return nil -} - -func (c *Core) Run(ctx context.Context, id uint64, _ models.RunOptions) models.AppError { - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return models.AppError{AppErrorType: models.ErrInternal, Err: err} - } - - runAppErrGrp, runAppCtx := errgroup.WithContext(ctx) - - inodeErrCh := make(chan error, 1) - appErrCh := make(chan models.AppError, 1) - inodeChan := make(chan uint64, 1) //send inode to the hook - - defer func() { - err := runAppErrGrp.Wait() - defer close(inodeErrCh) - defer close(inodeChan) - if err != nil { - utils.LogError(c.logger, err, "failed to stop the app") - } - }() - - runAppErrGrp.Go(func() error { - defer utils.Recover(c.logger) - if a.Kind(ctx) == utils.Native { - return nil - } - select { - case inode := <-inodeChan: - err := c.Hooks.SendDockerAppInfo(id, structs.DockerAppInfo{AppInode: inode, ClientID: id}) - if err != nil { - utils.LogError(c.logger, err, "") - - inodeErrCh <- errors.New("failed to send inode to the kernel") - } - case <-ctx.Done(): - return nil - } - return nil - }) - - runAppErrGrp.Go(func() error { - defer utils.Recover(c.logger) - defer close(appErrCh) - appErr := a.Run(runAppCtx, inodeChan) - if appErr.Err != nil { - utils.LogError(c.logger, appErr.Err, "error while running the app") - appErrCh <- appErr - } - return nil - }) - - select { - case <-runAppCtx.Done(): - return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: nil} - case appErr := <-appErrCh: - return appErr - case inodeErr := <-inodeErrCh: - return models.AppError{AppErrorType: models.ErrInternal, Err: inodeErr} - } -} - -func (c *Core) GetContainerIP(_ context.Context, id uint64) (string, error) { - - a, err := c.getApp(id) - if err != nil { - utils.LogError(c.logger, err, "failed to get app") - return "", err - } - - ip := a.ContainerIPv4Addr() - c.logger.Debug("ip address of the target app container", zap.Any("ip", ip)) - if ip == "" { - return "", fmt.Errorf("failed to get the IP address of the app container. Try increasing --delay (in seconds)") - } - - return ip, nil -} diff --git a/pkg/core/hooks/bpf_arm64_bpfel.go b/pkg/core/hooks/bpf_arm64_bpfel.go index df9cc8dbec..af17c81b4a 100644 --- a/pkg/core/hooks/bpf_arm64_bpfel.go +++ b/pkg/core/hooks/bpf_arm64_bpfel.go @@ -95,11 +95,11 @@ type bpfMapSpecs struct { ConnInfoMap *ebpf.MapSpec `ebpf:"conn_info_map"` CurrentSockMap *ebpf.MapSpec `ebpf:"current_sock_map"` DestInfoMap *ebpf.MapSpec `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.MapSpec `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.MapSpec `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.MapSpec `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.MapSpec `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.MapSpec `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.MapSpec `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.MapSpec `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.MapSpec `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.MapSpec `ebpf:"socket_data_event_buffer_heap"` @@ -135,11 +135,11 @@ type bpfMaps struct { ConnInfoMap *ebpf.Map `ebpf:"conn_info_map"` CurrentSockMap *ebpf.Map `ebpf:"current_sock_map"` DestInfoMap *ebpf.Map `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.Map `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.Map `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.Map `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.Map `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.Map `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.Map `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.Map `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.Map `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.Map `ebpf:"socket_data_event_buffer_heap"` @@ -158,11 +158,11 @@ func (m *bpfMaps) Close() error { m.ConnInfoMap, m.CurrentSockMap, m.DestInfoMap, - m.DockerAppRegistrationMap, m.KeployAgentKernelPidMap, m.KeployAgentRegistrationMap, m.KeployClientKernelPidMap, m.KeployClientRegistrationMap, + m.KeployProxyInfo, m.RedirectProxyMap, m.SocketCloseEvents, m.SocketDataEventBufferHeap, diff --git a/pkg/core/hooks/bpf_arm64_bpfel.o b/pkg/core/hooks/bpf_arm64_bpfel.o index f8c9fe223f..b7f61342eb 100644 Binary files a/pkg/core/hooks/bpf_arm64_bpfel.o and b/pkg/core/hooks/bpf_arm64_bpfel.o differ diff --git a/pkg/core/hooks/bpf_x86_bpfel.go b/pkg/core/hooks/bpf_x86_bpfel.go index cf8f135116..99ef0483c7 100644 --- a/pkg/core/hooks/bpf_x86_bpfel.go +++ b/pkg/core/hooks/bpf_x86_bpfel.go @@ -95,11 +95,11 @@ type bpfMapSpecs struct { ConnInfoMap *ebpf.MapSpec `ebpf:"conn_info_map"` CurrentSockMap *ebpf.MapSpec `ebpf:"current_sock_map"` DestInfoMap *ebpf.MapSpec `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.MapSpec `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.MapSpec `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.MapSpec `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.MapSpec `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.MapSpec `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.MapSpec `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.MapSpec `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.MapSpec `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.MapSpec `ebpf:"socket_data_event_buffer_heap"` @@ -135,11 +135,11 @@ type bpfMaps struct { ConnInfoMap *ebpf.Map `ebpf:"conn_info_map"` CurrentSockMap *ebpf.Map `ebpf:"current_sock_map"` DestInfoMap *ebpf.Map `ebpf:"dest_info_map"` - DockerAppRegistrationMap *ebpf.Map `ebpf:"docker_app_registration_map"` KeployAgentKernelPidMap *ebpf.Map `ebpf:"keploy_agent_kernel_pid_map"` KeployAgentRegistrationMap *ebpf.Map `ebpf:"keploy_agent_registration_map"` KeployClientKernelPidMap *ebpf.Map `ebpf:"keploy_client_kernel_pid_map"` KeployClientRegistrationMap *ebpf.Map `ebpf:"keploy_client_registration_map"` + KeployProxyInfo *ebpf.Map `ebpf:"keploy_proxy_info"` RedirectProxyMap *ebpf.Map `ebpf:"redirect_proxy_map"` SocketCloseEvents *ebpf.Map `ebpf:"socket_close_events"` SocketDataEventBufferHeap *ebpf.Map `ebpf:"socket_data_event_buffer_heap"` @@ -158,11 +158,11 @@ func (m *bpfMaps) Close() error { m.ConnInfoMap, m.CurrentSockMap, m.DestInfoMap, - m.DockerAppRegistrationMap, m.KeployAgentKernelPidMap, m.KeployAgentRegistrationMap, m.KeployClientKernelPidMap, m.KeployClientRegistrationMap, + m.KeployProxyInfo, m.RedirectProxyMap, m.SocketCloseEvents, m.SocketDataEventBufferHeap, diff --git a/pkg/core/hooks/bpf_x86_bpfel.o b/pkg/core/hooks/bpf_x86_bpfel.o index f3f76af725..3a45e1dfba 100644 Binary files a/pkg/core/hooks/bpf_x86_bpfel.o and b/pkg/core/hooks/bpf_x86_bpfel.o differ diff --git a/pkg/core/hooks/conn/socket.go b/pkg/core/hooks/conn/socket.go index d7af88596f..ff87e8b64b 100644 --- a/pkg/core/hooks/conn/socket.go +++ b/pkg/core/hooks/conn/socket.go @@ -9,6 +9,8 @@ import ( "errors" "fmt" "os" + "os/signal" + "syscall" "time" "unsafe" @@ -38,6 +40,38 @@ func ListenSocket(ctx context.Context, l *zap.Logger, openMap, dataMap, closeMap if !ok { return nil, errors.New("failed to get the error group from the context") } + + // Create a channel to listen for signals + sigChan := make(chan os.Signal, 1) + + // Register the signals you want to listen to (e.g., SIGINT, SIGTERM) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + go func() { + for { + println("Waiting for signal") + select { + case sig := <-sigChan: + // Handle the signal + fmt.Printf("Received signal::: %s\n", sig) + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + fmt.Println("Shutting down gracefully...") + return + default: + fmt.Printf("Unhandled signal::: %s\n", sig) + } + case <-ctx.Done(): + println("Context done") + // Exit the goroutine when the context is canceled + return + } + } + }() + + // Simulate some work being done + fmt.Println("Application running... Press Ctrl+C to stop.") + g.Go(func() error { defer utils.Recover(l) go func() { diff --git a/pkg/core/hooks/hooks.go b/pkg/core/hooks/hooks.go index 2f589abf69..f7183e42a9 100644 --- a/pkg/core/hooks/hooks.go +++ b/pkg/core/hooks/hooks.go @@ -28,6 +28,9 @@ import ( ) func NewHooks(logger *zap.Logger, cfg *config.Config) *Hooks { + if cfg.Agent.IsDocker { + cfg.ProxyPort = 36789 + } return &Hooks{ logger: logger, sess: core.NewSessions(), @@ -49,10 +52,11 @@ type Hooks struct { m sync.Mutex // eBPF C shared maps - clientRegistrationMap *ebpf.Map - agentRegistartionMap *ebpf.Map - dockerAppRegistrationMap *ebpf.Map - redirectProxyMap *ebpf.Map + clientRegistrationMap *ebpf.Map + agentRegistartionMap *ebpf.Map + + redirectProxyMap *ebpf.Map + proxyInfoMap *ebpf.Map //-------------- // eBPF C shared objectsobjects @@ -87,7 +91,6 @@ type Hooks struct { objects bpfObjects writev link.Link writevRet link.Link - appID uint64 } func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { @@ -96,7 +99,7 @@ func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { ID: id, }) - err := h.load(ctx, opts) + err := h.load(opts) if err != nil { return err } @@ -119,7 +122,7 @@ func (h *Hooks) Load(ctx context.Context, id uint64, opts core.HookCfg) error { return nil } -func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { +func (h *Hooks) load(opts core.HookCfg) error { // Allow the current process to lock memory for eBPF resources. if err := rlimit.RemoveMemlock(); err != nil { utils.LogError(h.logger, err, "failed to lock memory for eBPF resources") @@ -142,7 +145,8 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.redirectProxyMap = objs.RedirectProxyMap h.clientRegistrationMap = objs.KeployClientRegistrationMap h.agentRegistartionMap = objs.KeployAgentRegistrationMap - h.dockerAppRegistrationMap = objs.DockerAppRegistrationMap + h.proxyInfoMap = objs.KeployProxyInfo + h.objects = objs // --------------- @@ -408,28 +412,17 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.logger.Info("keploy initialized and probes added to the kernel.") - var clientInfo structs.ClientInfo = structs.ClientInfo{} - - switch opts.Mode { - case models.MODE_RECORD: - clientInfo.Mode = uint32(1) - case models.MODE_TEST: - clientInfo.Mode = uint32(2) - default: - clientInfo.Mode = uint32(0) - } - //sending keploy pid to kernel to get filtered - inode, err := getSelfInodeNumber() - if err != nil { - utils.LogError(h.logger, err, "failed to get inode of the keploy process") - return err - } + // inode, err := getSelfInodeNumber() + // if err != nil { + // utils.LogError(h.logger, err, "failed to get inode of the keploy process") + // return err + // } - clientInfo.KeployClientInode = inode - clientInfo.KeployClientNsPid = uint32(os.Getpid()) - clientInfo.IsKeployClientRegistered = uint32(0) - h.logger.Debug("Keploy Pid sent successfully...") + // clientInfo.KeployClientInode = inode + // clientInfo.KeployClientNsPid = uint32(os.Getpid()) + // clientInfo.IsKeployClientRegistered = uint32(0) + // h.logger.Info("Keploy Pid sent successfully...") if opts.IsDocker { h.proxyIP4 = opts.KeployIPV4 @@ -443,41 +436,38 @@ func (h *Hooks) load(ctx context.Context, opts core.HookCfg) error { h.logger.Debug("proxy ips", zap.String("ipv4", h.proxyIP4), zap.Any("ipv6", h.proxyIP6)) - proxyIP, err := IPv4ToUint32(h.proxyIP4) - if err != nil { - return fmt.Errorf("failed to convert ip string:[%v] to 32-bit integer", opts.KeployIPV4) - } - var agentInfo structs.AgentInfo = structs.AgentInfo{} - - agentInfo.ProxyInfo = structs.ProxyInfo{ - IP4: proxyIP, - IP6: h.proxyIP6, - Port: h.proxyPort, + agentInfo.KeployAgentNsPid = uint32(os.Getpid()) + agentInfo.KeployAgentInode, err = GetSelfInodeNumber() + if err != nil { + utils.LogError(h.logger, err, "failed to get inode of the keploy process") + return err } agentInfo.DNSPort = int32(h.dnsPort) - if opts.IsDocker { - clientInfo.IsDockerApp = uint32(1) - } else { - clientInfo.IsDockerApp = uint32(0) - } - - ports := GetPortToSendToKernel(ctx, opts.Rules) - for i := 0; i < 10; i++ { - if len(ports) <= i { - clientInfo.PassThroughPorts[i] = -1 - continue - } - clientInfo.PassThroughPorts[i] = int32(ports[i]) - } - - err = h.SendClientInfo(opts.AppID, clientInfo) - if err != nil { - h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) - return err - } + // if opts.IsDocker { + // clientInfo.IsDockerApp = uint32(1) + // } else { + // clientInfo.IsDockerApp = uint32(0) + // } + + // ports := GetPortToSendToKernel(ctx, opts.Rules) + // for i := 0; i < 10; i++ { + // if len(ports) <= i { + // clientInfo.PassThroughPorts[i] = -1 + // continue + // } + // clientInfo.PassThroughPorts[i] = int32(ports[i]) + // } + + // for sending client pid to kernel + // fmt.Println("Sending client info to kernel...", clientInfo) + // err = h.SendClientInfo(opts.AppID, clientInfo) + // if err != nil { + // h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + // return err + // } err = h.SendAgentInfo(agentInfo) if err != nil { h.logger.Error("failed to send agent info to the ebpf program", zap.Error(err)) @@ -491,9 +481,33 @@ func (h *Hooks) Record(ctx context.Context, _ uint64, opts models.IncomingOption // TODO use the session to get the app id // and then use the app id to get the test cases chan // and pass that to eBPF consumers/listeners + fmt.Println("Recording hooks...") return conn.ListenSocket(ctx, h.logger, h.objects.SocketOpenEvents, h.objects.SocketDataEvents, h.objects.SocketCloseEvents, opts) } +func (h *Hooks) SendKeployClientInfo(clientID uint64, clientInfo structs.ClientInfo) error { + // TODO use the session to get the app id + // and then use the app id to get the test cases chan + // and pass that to eBPF consumers/listeners + + err := h.SendClientInfo(clientID, clientInfo) + if err != nil { + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + return err + } + + return nil +} + +func (h *Hooks) SendClientProxyInfo(clientID uint64, proxyInfo structs.ProxyInfo) error { + err := h.SendProxyInfo(clientID, proxyInfo) + if err != nil { + h.logger.Error("failed to send app info to the ebpf program", zap.Error(err)) + return err + } + return nil +} + func (h *Hooks) unLoad(_ context.Context) { // closing all events //other diff --git a/pkg/core/hooks/kernelComm.go b/pkg/core/hooks/kernelComm.go index abf8d98d07..acdcb717aa 100644 --- a/pkg/core/hooks/kernelComm.go +++ b/pkg/core/hooks/kernelComm.go @@ -6,8 +6,6 @@ import ( "context" "fmt" - "math/rand" - "github.com/cilium/ebpf" "go.keploy.io/server/v2/pkg/core" "go.keploy.io/server/v2/pkg/core/hooks/structs" @@ -74,30 +72,21 @@ func (h *Hooks) SendClientInfo(id uint64, appInfo structs.ClientInfo) error { return nil } -func (h *Hooks) SendAgentInfo(agentInfo structs.AgentInfo) error { - key := 0 - err := h.agentRegistartionMap.Update(uint32(key), agentInfo, ebpf.UpdateAny) +// SendProxyInfo sends the network information to the kernel +func (h *Hooks) SendProxyInfo(id uint64, proxInfo structs.ProxyInfo) error { + err := h.proxyInfoMap.Update(id, proxInfo, ebpf.UpdateAny) if err != nil { - utils.LogError(h.logger, err, "failed to send the agent info to the ebpf program") + utils.LogError(h.logger, err, "failed to send the proxy info to the ebpf program") return err } return nil } -func (h *Hooks) SendDockerAppInfo(_ uint64, dockerAppInfo structs.DockerAppInfo) error { - if h.appID != 0 { - err := h.dockerAppRegistrationMap.Delete(h.appID) - if err != nil { - utils.LogError(h.logger, err, "failed to remove entry from dockerAppRegistrationMap") - return err - } - } - r := rand.New(rand.NewSource(rand.Int63())) - randomNum := r.Uint64() - h.appID = randomNum - err := h.dockerAppRegistrationMap.Update(h.appID, dockerAppInfo, ebpf.UpdateAny) +func (h *Hooks) SendAgentInfo(agentInfo structs.AgentInfo) error { + key := 0 + err := h.agentRegistartionMap.Update(uint32(key), agentInfo, ebpf.UpdateAny) if err != nil { - utils.LogError(h.logger, err, "failed to send the dockerAppInfo info to the ebpf program") + utils.LogError(h.logger, err, "failed to send the agent info to the ebpf program") return err } return nil diff --git a/pkg/core/hooks/structs/structs.go b/pkg/core/hooks/structs/structs.go index 3e9483d203..1546f2be79 100755 --- a/pkg/core/hooks/structs/structs.go +++ b/pkg/core/hooks/structs/structs.go @@ -5,15 +5,6 @@ package structs type BpfSpinLock struct{ Val uint32 } -// struct dest_info_t -// { -// u32 ip_version; -// u32 dest_ip4; -// u32 dest_ip6[4]; -// u32 dest_port; -// u32 kernelPid; -// }; - type DestInfo struct { IPVersion uint32 DestIP4 uint32 @@ -23,13 +14,6 @@ type DestInfo struct { ClientID uint64 } -// struct proxy_info -// { -// u32 ip4; -// u32 ip6[4]; -// u32 port; -// }; - type ProxyInfo struct { IP4 uint32 IP6 [4]uint32 @@ -41,17 +25,6 @@ type DockerAppInfo struct { ClientID uint64 } -// struct app_info -// { -// u32 keploy_client_ns_pid; -// u64 keploy_client_inode; -// u64 app_inode; -// u32 mode; -// u32 is_docker_app; -// u32 is_keploy_client_registered; // whether the client is registered or not -// s32 pass_through_ports[PASS_THROUGH_ARRAY_SIZE]; -// }; - type ClientInfo struct { KeployClientInode uint64 KeployClientNsPid uint32 @@ -59,19 +32,11 @@ type ClientInfo struct { IsDockerApp uint32 IsKeployClientRegistered uint32 PassThroughPorts [10]int32 + AppInode uint64 } -// struct agent_info -// { -// u32 keploy_agent_ns_pid; -// u32 keploy_agent_inode; -// struct proxy_info proxy_info; -// s32 dns_port; -// }; - type AgentInfo struct { KeployAgentNsPid uint32 DNSPort int32 KeployAgentInode uint64 - ProxyInfo ProxyInfo } diff --git a/pkg/core/hooks/util.go b/pkg/core/hooks/util.go index f28a9ff485..4c315941df 100644 --- a/pkg/core/hooks/util.go +++ b/pkg/core/hooks/util.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build !windows package hooks @@ -97,7 +97,7 @@ func detectCgroupPath(logger *zap.Logger) (string, error) { return "", errors.New("cgroup2 not mounted") } -func getSelfInodeNumber() (uint64, error) { +func GetSelfInodeNumber() (uint64, error) { p := filepath.Join("/proc", "self", "ns", "pid") f, err := os.Stat(p) diff --git a/pkg/core/proxy/integrations/mongo/mongo.go b/pkg/core/proxy/integrations/mongo/mongo.go index fefe6fea42..044f0d0b87 100644 --- a/pkg/core/proxy/integrations/mongo/mongo.go +++ b/pkg/core/proxy/integrations/mongo/mongo.go @@ -82,6 +82,7 @@ func (m *Mongo) MockOutgoing(ctx context.Context, src net.Conn, dstCfg *integrat return err } + m.logger.Info("Mocking the mongo message") // converts the yaml string into the binary packet err = decodeMongo(ctx, logger, reqBuf, src, dstCfg, mockDb, opts) if err != nil { diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go b/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go index 59d8b51a26..9def6aae9a 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go +++ b/pkg/core/proxy/integrations/mysql/wire/phase/conn/handshakeResponse41Packet.go @@ -10,6 +10,7 @@ import ( "fmt" "go.keploy.io/server/v2/pkg/core/proxy/integrations/mysql/utils" + "go.keploy.io/server/v2/pkg/core/proxy/integrations/util" "go.keploy.io/server/v2/pkg/models/mysql" "go.uber.org/zap" ) @@ -63,7 +64,7 @@ func DecodeHandshakeResponse41(_ context.Context, _ *zap.Logger, data []byte) (* if packet.CapabilityFlags&mysql.CLIENT_CONNECT_WITH_DB != 0 { idx = bytes.IndexByte(data, 0x00) if idx != -1 { - packet.Database = string(data[:idx]) + packet.Database = util.EncodeBase64(data[:idx]) data = data[idx+1:] } } @@ -73,7 +74,7 @@ func DecodeHandshakeResponse41(_ context.Context, _ *zap.Logger, data []byte) (* if idx == -1 { return nil, errors.New("malformed handshake response packet: missing null terminator for AuthPluginName") } - packet.AuthPluginName = string(data[:idx]) + packet.AuthPluginName = util.EncodeBase64(data[:idx]) data = data[idx+1:] } diff --git a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go b/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go index a985fc62ac..df858e2579 100644 --- a/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go +++ b/pkg/core/proxy/integrations/mysql/wire/phase/query/rowscols/binaryProtocolRowPacket.go @@ -255,14 +255,37 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow columnEntry := row.Values[i] + fmt.Printf("ColumnEntry: %+v\n", columnEntry) + fmt.Printf("columnEntry.Value: %+v\n", columnEntry.Value) + fmt.Printf("Type of Value: %T\n", columnEntry.Value) + switch columnEntry.Type { case mysql.FieldTypeLong: var val any if columnEntry.Unsigned { - val = uint32(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint32(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint32(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint32(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint32(columnEntry.Value.(uint)) + } } else { - val = int32(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int32(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int32(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int32(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int32(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } @@ -277,10 +300,29 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow case mysql.FieldTypeTiny: var val any if columnEntry.Unsigned { - val = uint8(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint8(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint8(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint8(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint8(columnEntry.Value.(uint)) + } } else { - val = int8(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int8(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int8(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int8(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int8(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } @@ -288,21 +330,60 @@ func EncodeBinaryRow(_ context.Context, logger *zap.Logger, row *mysql.BinaryRow case mysql.FieldTypeShort, mysql.FieldTypeYear: var val any if columnEntry.Unsigned { - val = uint16(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint16(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint16(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint16(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint16(columnEntry.Value.(uint)) + } + } else { - val = int16(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int16(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int16(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int16(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int16(columnEntry.Value.(uint)) + } } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write int16 value: %w", err) } case mysql.FieldTypeLongLong: var val any if columnEntry.Unsigned { - val = uint64(columnEntry.Value.(int)) + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = uint64(columnEntry.Value.(float64)) + case int: + columnEntry.Value = uint64(columnEntry.Value.(int)) + case int64: + columnEntry.Value = uint64(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = uint64(columnEntry.Value.(uint)) + } } else { - val = int64(columnEntry.Value.(int)) - } + switch columnEntry.Value.(type) { + case float64: + columnEntry.Value = int64(columnEntry.Value.(float64)) + case int: + columnEntry.Value = int64(columnEntry.Value.(int)) + case int64: + columnEntry.Value = int64(columnEntry.Value.(int64)) + case uint: + columnEntry.Value = int64(columnEntry.Value.(uint)) + } + } + val = columnEntry.Value if err := binary.Write(buf, binary.LittleEndian, val); err != nil { return nil, fmt.Errorf("failed to write %T value: %w", val, err) } diff --git a/pkg/core/proxy/integrations/postgres/v1/decode.go b/pkg/core/proxy/integrations/postgres/v1/decode.go index cc395af644..224b380458 100644 --- a/pkg/core/proxy/integrations/postgres/v1/decode.go +++ b/pkg/core/proxy/integrations/postgres/v1/decode.go @@ -67,7 +67,7 @@ func decodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie } if !matched { - logger.Debug("MISMATCHED REQ is" + string(pgRequests[0])) + logger.Info("MISMATCHED REQ is" + string(pgRequests[0])) _, err = pUtil.PassThrough(ctx, logger, clientConn, dstCfg, pgRequests) if err != nil { utils.LogError(logger, err, "failed to pass the request", zap.Any("request packets", len(pgRequests))) diff --git a/pkg/core/proxy/integrations/postgres/v1/encode.go b/pkg/core/proxy/integrations/postgres/v1/encode.go index 90b0d739d6..dda0c72208 100755 --- a/pkg/core/proxy/integrations/postgres/v1/encode.go +++ b/pkg/core/proxy/integrations/postgres/v1/encode.go @@ -329,7 +329,6 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie PacketTypes: pg.FrontendWrapper.PacketTypes, Identfier: "ServerResponse", Length: uint32(len(reqBuf)), - // Payload: bufStr, AuthenticationOk: pg.FrontendWrapper.AuthenticationOk, AuthenticationCleartextPassword: pg.FrontendWrapper.AuthenticationCleartextPassword, AuthenticationMD5Password: pg.FrontendWrapper.AuthenticationMD5Password, @@ -345,8 +344,8 @@ func encodePostgres(ctx context.Context, logger *zap.Logger, reqBuf []byte, clie CommandCompletes: pg.FrontendWrapper.CommandCompletes, CopyData: pg.FrontendWrapper.CopyData, CopyDone: pg.FrontendWrapper.CopyDone, - CopyInResponse: pg.FrontendWrapper.CopyInResponse, - CopyOutResponse: pg.FrontendWrapper.CopyOutResponse, + // CopyInResponse: pg.FrontendWrapper.CopyInResponse, + // CopyOutResponse: pg.FrontendWrapper.CopyOutResponse, DataRow: pg.FrontendWrapper.DataRow, DataRows: pg.FrontendWrapper.DataRows, EmptyQueryResponse: pg.FrontendWrapper.EmptyQueryResponse, diff --git a/pkg/core/proxy/integrations/postgres/v1/match.go b/pkg/core/proxy/integrations/postgres/v1/match.go index 819a9bd35d..ffe14acadc 100644 --- a/pkg/core/proxy/integrations/postgres/v1/match.go +++ b/pkg/core/proxy/integrations/postgres/v1/match.go @@ -93,7 +93,6 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut reqGoingOn := decodePgRequest(requestBuffers[0], logger) if reqGoingOn != nil { logger.Debug("PacketTypes", zap.Any("PacketTypes", reqGoingOn.PacketTypes)) - // fmt.Println("REQUEST GOING ON - ", reqGoingOn) logger.Debug("ConnectionId-", zap.String("ConnectionId", ConnectionID)) logger.Debug("TestMap*****", zap.Any("TestMap", testmap)) } @@ -142,7 +141,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut } return true, []models.Frontend{ssl}, nil case initMock.Spec.PostgresRequests[requestIndex].Identfier == "StartupRequest" && isStartupPacket(reqBuff) && initMock.Spec.PostgresRequests[requestIndex].Payload != "AAAACATSFi8=" && initMock.Spec.PostgresResponses[requestIndex].AuthType == 10: - logger.Debug("CHANGING TO MD5 for Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) + logger.Info("CHANGING TO MD5 for Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) res := make([]models.Frontend, len(initMock.Spec.PostgresResponses)) copy(res, initMock.Spec.PostgresResponses) res[requestIndex].AuthType = 5 @@ -152,7 +151,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut } return true, res, nil case len(encodedMock) > 0 && encodedMock[0] == 'p' && initMock.Spec.PostgresRequests[requestIndex].PacketTypes[0] == "p" && reqBuff[0] == 'p': - logger.Debug("CHANGING TO MD5 for Request and Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) + logger.Info("CHANGING TO MD5 for Request and Response", zap.String("mock", initMock.Name), zap.String("Req", bufStr)) res := make([]models.Frontend, len(initMock.Spec.PostgresResponses)) copy(res, initMock.Spec.PostgresResponses) @@ -223,7 +222,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut getTestPS(requestBuffers, logger, ConnectionID) } - logger.Debug("Sorted Mocks inside pg parser: ", zap.Any("Len of sortedTcsMocks", len(sortedTcsMocks))) + logger.Info("Sorted Mocks inside pg parser: ", zap.Any("Len of sortedTcsMocks", len(sortedTcsMocks))) var matched, sorted bool var idx int @@ -238,7 +237,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut if newMock != nil { matchedMock = newMock } - logger.Debug("Matched In Sorted PG Matching Stream", zap.String("mock", matchedMock.Name)) + logger.Info("Matched In Sorted PG Matching Stream", zap.String("mock", matchedMock.Name)) } idx = findBinaryStreamMatch(logger, sortedTcsMocks, requestBuffers, sorted) @@ -257,7 +256,7 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut if newMock != nil { matchedMock = newMock } - logger.Debug("Matched In Unsorted PG Matching Stream", zap.String("mock", matchedMock.Name)) + logger.Info("Matched In Unsorted PG Matching Stream", zap.String("mock", matchedMock.Name)) } idx = findBinaryStreamMatch(logger, tcsMocks, requestBuffers, sorted) // check if the validate the query with the matched mock @@ -273,12 +272,12 @@ func matchingReadablePG(ctx context.Context, logger *zap.Logger, mutex *sync.Mut if newMock != nil && !isValid { matchedMock = newMock } - logger.Debug("Matched In Binary Matching for Unsorted", zap.String("mock", matchedMock.Name)) + logger.Info("Matched In Binary Matching for Unsorted", zap.String("mock", matchedMock.Name)) } } if matched { - logger.Debug("Matched mock", zap.String("mock", matchedMock.Name)) + logger.Info("Matched mock", zap.String("mock", matchedMock.Name)) if matchedMock.TestModeInfo.IsFiltered { originalMatchedMock := *matchedMock matchedMock.TestModeInfo.IsFiltered = false @@ -350,13 +349,13 @@ func findBinaryStreamMatch(logger *zap.Logger, tcsMocks []*models.Mock, requestB if sorted { if mxIdx != -1 && mxSim >= 0.78 { - logger.Debug("Matched with Sorted Stream", zap.Float64("similarity", mxSim)) + logger.Info("Matched with Sorted Stream", zap.Float64("similarity", mxSim)) } else { mxIdx = -1 } } else { if mxIdx != -1 { - logger.Debug("Matched with Unsorted Stream", zap.Float64("similarity", mxSim)) + logger.Info("Matched with Unsorted Stream", zap.Float64("similarity", mxSim)) } } return mxIdx diff --git a/pkg/core/proxy/integrations/postgres/v1/transcoder.go b/pkg/core/proxy/integrations/postgres/v1/transcoder.go index 9e1ce08c9a..e84158b7d1 100644 --- a/pkg/core/proxy/integrations/postgres/v1/transcoder.go +++ b/pkg/core/proxy/integrations/postgres/v1/transcoder.go @@ -111,9 +111,9 @@ func (f *FrontendWrapper) translateToReadableResponse(logger *zap.Logger, msgBod case 'E': msg = &f.FrontendWrapper.ErrorResponse case 'G': - msg = &f.FrontendWrapper.CopyInResponse - case 'H': - msg = &f.FrontendWrapper.CopyOutResponse + // msg = &f.FrontendWrapper.CopyInResponse + // case 'H': + // msg = &f.FrontendWrapper.CopyOutResponse case 'I': msg = &f.FrontendWrapper.EmptyQueryResponse case 'K': @@ -138,8 +138,6 @@ func (f *FrontendWrapper) translateToReadableResponse(logger *zap.Logger, msgBod msg = &f.FrontendWrapper.RowDescription case 'V': msg = &f.FrontendWrapper.FunctionCallResponse - case 'W': - msg = &f.FrontendWrapper.CopyBothResponse case 'Z': msg = &f.FrontendWrapper.ReadyForQuery default: diff --git a/pkg/core/proxy/integrations/postgres/v1/util.go b/pkg/core/proxy/integrations/postgres/v1/util.go index e0a385cdc1..64f4a949d6 100755 --- a/pkg/core/proxy/integrations/postgres/v1/util.go +++ b/pkg/core/proxy/integrations/postgres/v1/util.go @@ -78,16 +78,16 @@ func postgresDecoderFrontend(response models.Frontend) ([]byte, error) { Line: response.ErrorResponse.Line, Routine: response.ErrorResponse.Routine, } - case string('G'): - msg = &pgproto3.CopyInResponse{ - OverallFormat: response.CopyInResponse.OverallFormat, - ColumnFormatCodes: response.CopyInResponse.ColumnFormatCodes, - } - case string('H'): - msg = &pgproto3.CopyOutResponse{ - OverallFormat: response.CopyOutResponse.OverallFormat, - ColumnFormatCodes: response.CopyOutResponse.ColumnFormatCodes, - } + // case string('G'): + // msg = &pgproto3.CopyInResponse{ + // OverallFormat: response.CopyInResponse.OverallFormat, + // ColumnFormatCodes: response.CopyInResponse.ColumnFormatCodes, + // } + // case string('H'): + // msg = &pgproto3.CopyOutResponse{ + // OverallFormat: response.CopyOutResponse.OverallFormat, + // ColumnFormatCodes: response.CopyOutResponse.ColumnFormatCodes, + // } case string('I'): msg = &pgproto3.EmptyQueryResponse{} case string('K'): @@ -165,11 +165,11 @@ func postgresDecoderFrontend(response models.Frontend) ([]byte, error) { msg = &pgproto3.FunctionCallResponse{ Result: response.FunctionCallResponse.Result, } - case string('W'): - msg = &pgproto3.CopyBothResponse{ - OverallFormat: response.CopyBothResponse.OverallFormat, - ColumnFormatCodes: response.CopyBothResponse.ColumnFormatCodes, - } + // case string('W'): + // msg = &pgproto3.CopyBothResponse{ + // // OverallFormat: response.CopyBothResponse.OverallFormat, + // ColumnFormatCodes: response.CopyBothResponse.ColumnFormatCodes, + // } case string('Z'): msg = &pgproto3.ReadyForQuery{ TxStatus: response.ReadyForQuery.TxStatus, diff --git a/pkg/core/proxy/proxy.go b/pkg/core/proxy/proxy.go index dbcfed125c..a423389eca 100755 --- a/pkg/core/proxy/proxy.go +++ b/pkg/core/proxy/proxy.go @@ -57,6 +57,10 @@ type Proxy struct { } func New(logger *zap.Logger, info core.DestInfo, opts *config.Config) *Proxy { + if opts.Agent.IsDocker { + logger.Info("Running in docker environment proxy port will be set to 36789") + opts.ProxyPort = 36789 + } return &Proxy{ logger: logger, Port: opts.ProxyPort, // default: 16789 @@ -89,7 +93,6 @@ func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { utils.LogError(p.logger, err, "failed to initialize the integrations") return err } - // set up the CA for tls connections err = SetupCA(ctx, p.logger) if err != nil { @@ -100,11 +103,14 @@ func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { if !ok { return errors.New("failed to get the error group from the context") } + // Create a channel to signal readiness of each server + readyChan := make(chan error, 1) // start the proxy server g.Go(func() error { defer utils.Recover(p.logger) - err := p.start(ctx) + err := p.start(ctx, readyChan) + readyChan <- err if err != nil { utils.LogError(p.logger, err, "error while running the proxy server") return err @@ -172,6 +178,11 @@ func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { return err } }) + // Wait for the proxy server to be ready or fail + err = <-readyChan + if err != nil { + return err + } p.logger.Info("Keploy has taken control of the DNS resolution mechanism, your application may misbehave if you have provided wrong domain name in your application code.") p.logger.Info(fmt.Sprintf("Proxy started at port:%v", p.Port)) @@ -179,17 +190,21 @@ func (p *Proxy) StartProxy(ctx context.Context, opts core.ProxyOptions) error { } // start function starts the proxy server on the idle local port -func (p *Proxy) start(ctx context.Context) error { +func (p *Proxy) start(ctx context.Context, readyChan chan<- error) error { // It will listen on all the interfaces listener, err := net.Listen("tcp", fmt.Sprintf(":%v", p.Port)) if err != nil { utils.LogError(p.logger, err, fmt.Sprintf("failed to start proxy on port:%v", p.Port)) + // Notify failure + readyChan <- err return err } + p.Listener = listener p.logger.Debug(fmt.Sprintf("Proxy server is listening on %s", fmt.Sprintf(":%v", listener.Addr()))) - + // Signal that the server is ready + readyChan <- nil defer func(listener net.Listener) { err := listener.Close() @@ -205,7 +220,7 @@ func (p *Proxy) start(ctx context.Context) error { clientConnCancel() err := clientConnErrGrp.Wait() if err != nil { - p.logger.Debug("failed to handle the client connection", zap.Error(err)) + p.logger.Info("failed to handle the client connection", zap.Error(err)) } //closing all the mock channels (if any in record mode) for _, mc := range p.sessions.GetAllMC() { @@ -280,8 +295,7 @@ func (p *Proxy) handleConnection(ctx context.Context, srcConn net.Conn) error { remoteAddr := srcConn.RemoteAddr().(*net.TCPAddr) sourcePort := remoteAddr.Port - p.logger.Debug("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) - + p.logger.Info("Inside handleConnection of proxyServer", zap.Any("source port", sourcePort), zap.Any("Time", time.Now().Unix())) destInfo, err := p.DestInfo.Get(ctx, uint16(sourcePort)) if err != nil { utils.LogError(p.logger, err, "failed to fetch the destination info", zap.Any("Source port", sourcePort)) @@ -570,7 +584,6 @@ func (p *Proxy) Record(_ context.Context, id uint64, mocks chan<- *models.Mock, MC: mocks, OutgoingOptions: opts, }) - p.MockManagers.Store(id, NewMockManager(NewTreeDb(customComparator), NewTreeDb(customComparator), p.logger)) ////set the new proxy ip:port for a new session @@ -622,7 +635,6 @@ func (p *Proxy) SetMocks(_ context.Context, id uint64, filtered []*models.Mock, m.(*MockManager).SetFilteredMocks(filtered) m.(*MockManager).SetUnFilteredMocks(unFiltered) } - return nil } diff --git a/pkg/core/record.go b/pkg/core/record.go deleted file mode 100644 index 471ebf40e5..0000000000 --- a/pkg/core/record.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build linux - -package core - -import ( - "context" - - "go.keploy.io/server/v2/pkg/models" -) - -func (c *Core) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { - return c.Hooks.Record(ctx, id, opts) -} - -func (c *Core) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { - m := make(chan *models.Mock, 500) - - err := c.Proxy.Record(ctx, id, m, opts) - if err != nil { - return nil, err - } - - return m, nil -} diff --git a/pkg/core/replay.go b/pkg/core/replay.go deleted file mode 100644 index 4bc6cfdd6e..0000000000 --- a/pkg/core/replay.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build linux - -package core - -import ( - "context" - - "go.keploy.io/server/v2/pkg/models" -) - -func (c *Core) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { - - err := c.Proxy.Mock(ctx, id, opts) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/core/service.go b/pkg/core/service.go index 27c43d835f..894f6add54 100644 --- a/pkg/core/service.go +++ b/pkg/core/service.go @@ -15,11 +15,13 @@ import ( ) type Hooks interface { - AppInfo DestInfo OutgoingInfo Load(ctx context.Context, id uint64, cfg HookCfg) error Record(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) + // send KeployClient Pid + SendKeployClientInfo(clientID uint64, clientInfo structs.ClientInfo) error + SendClientProxyInfo(clientID uint64, proxyInfo structs.ProxyInfo) error } type HookCfg struct { @@ -33,7 +35,7 @@ type HookCfg struct { type App interface { Setup(ctx context.Context, opts app.Options) error - Run(ctx context.Context, inodeChan chan uint64, opts app.Options) error + Run(ctx context.Context, opts app.Options) error Kind(ctx context.Context) utils.CmdType KeployIPv4Addr() string } @@ -59,10 +61,6 @@ type DestInfo interface { Delete(ctx context.Context, srcPort uint16) error } -type AppInfo interface { - SendDockerAppInfo(id uint64, dockerAppInfo structs.DockerAppInfo) error -} - // For keploy test bench type Tester interface { diff --git a/pkg/models/agent.go b/pkg/models/agent.go new file mode 100644 index 0000000000..4ec89cef0b --- /dev/null +++ b/pkg/models/agent.go @@ -0,0 +1,32 @@ +package models + +type OutgoingReq struct { + OutgoingOptions OutgoingOptions `json:"outgoingOptions"` + ClientID uint64 `json:"clientId"` +} + +type IncomingReq struct { + IncomingOptions IncomingOptions `json:"incomingOptions"` + ClientID uint64 `json:"clientId"` +} + +type RegisterReq struct { + SetupOptions SetupOptions `json:"setupOptions"` +} + +type AgentResp struct { + ClientID int64 `json:"clientID"` // uuid of the app + Error error `json:"error"` + IsSuccess bool `json:"isSuccess"` +} + +type RunReq struct { + RunOptions RunOptions `json:"runOptions"` + ClientID uint64 `json:"clientId"` +} + +type SetMocksReq struct { + Filtered []*Mock `json:"filtered"` + UnFiltered []*Mock `json:"unFiltered"` + ClientID uint64 `json:"clientId"` +} diff --git a/pkg/models/instrument.go b/pkg/models/instrument.go index 8f38f165c5..7cefacd598 100644 --- a/pkg/models/instrument.go +++ b/pkg/models/instrument.go @@ -10,6 +10,7 @@ type HookOptions struct { Rules []config.BypassRule Mode Mode EnableTesting bool + IsDocker bool } type OutgoingOptions struct { @@ -26,9 +27,17 @@ type IncomingOptions struct { } type SetupOptions struct { + ClientID uint64 Container string DockerNetwork string DockerDelay uint64 + ClientNsPid uint32 + ClientInode uint64 + AppInode uint64 + Cmd string + IsDocker bool + CommandType string + Mode Mode } type RunOptions struct { diff --git a/pkg/models/mysql/comm.go b/pkg/models/mysql/comm.go index c0ed297ca8..2dfef943f4 100644 --- a/pkg/models/mysql/comm.go +++ b/pkg/models/mysql/comm.go @@ -1,4 +1,4 @@ -// Package mysql in models provides realted structs for mysql protocol +// Package mysql in models provides related structs for mysql protocol package mysql // This file contains struct for command phase packets @@ -13,157 +13,157 @@ package mysql // COM_QUERY packet (currently does not support if CLIENT_QUERY_ATTRIBUTES is set) type QueryPacket struct { - Command byte `yaml:"command"` - Query string `yaml:"query"` + Command byte `yaml:"command" json:"command"` + Query string `yaml:"query" json:"query"` } // LocalInFileRequestPacket is used to send local file request to server, currently not supported type LocalInFileRequestPacket struct { - PacketType byte `yaml:"command"` - Filename string + PacketType byte `yaml:"command" json:"command"` + Filename string `yaml:"filename" json:"filename"` } // TextResultSet is used as a response packet for COM_QUERY type TextResultSet struct { - ColumnCount uint64 `yaml:"columnCount"` - Columns []*ColumnDefinition41 `yaml:"columns"` - EOFAfterColumns []byte `yaml:"eofAfterColumns"` - Rows []*TextRow `yaml:"rows"` - FinalResponse *GenericResponse `yaml:"FinalResponse"` + ColumnCount uint64 `yaml:"columnCount" json:"columnCount"` + Columns []*ColumnDefinition41 `yaml:"columns" json:"columns"` + EOFAfterColumns []byte `yaml:"eofAfterColumns" json:"eofAfterColumns"` + Rows []*TextRow `yaml:"rows" json:"rows"` + FinalResponse *GenericResponse `yaml:"FinalResponse" json:"FinalResponse"` } // BinaryProtocolResultSet is used as a response packet for COM_STMT_EXECUTE type BinaryProtocolResultSet struct { - ColumnCount uint64 `yaml:"columnCount"` - Columns []*ColumnDefinition41 `yaml:"columns"` - EOFAfterColumns []byte `yaml:"eofAfterColumns"` - Rows []*BinaryRow `yaml:"rows"` - FinalResponse *GenericResponse `yaml:"FinalResponse"` + ColumnCount uint64 `yaml:"columnCount" json:"columnCount"` + Columns []*ColumnDefinition41 `yaml:"columns" json:"columns"` + EOFAfterColumns []byte `yaml:"eofAfterColumns" json:"eofAfterColumns"` + Rows []*BinaryRow `yaml:"rows" json:"rows"` + FinalResponse *GenericResponse `yaml:"FinalResponse" json:"FinalResponse"` } type GenericResponse struct { - Data []byte `yaml:"data"` - Type string `yaml:"type"` + Data []byte `yaml:"data" json:"data"` + Type string `yaml:"type" json:"type"` } // Columns type ColumnCount struct { - // Header Header `yaml:"header"` - Count uint64 `yaml:"count"` + // Header Header `yaml:"header" json:"header"` + Count uint64 `yaml:"count" json:"count"` } type ColumnDefinition41 struct { - Header Header `yaml:"header"` - Catalog string `yaml:"catalog"` - Schema string `yaml:"schema"` - Table string `yaml:"table"` - OrgTable string `yaml:"org_table"` - Name string `yaml:"name"` - OrgName string `yaml:"org_name"` - FixedLength byte `yaml:"fixed_length"` - CharacterSet uint16 `yaml:"character_set"` - ColumnLength uint32 `yaml:"column_length"` - Type byte `yaml:"type"` - Flags uint16 `yaml:"flags"` - Decimals byte `yaml:"decimals"` - Filler []byte `yaml:"filler"` - DefaultValue string `yaml:"defaultValue"` -} - -//Rows + Header Header `yaml:"header" json:"header"` + Catalog string `yaml:"catalog" json:"catalog"` + Schema string `yaml:"schema" json:"schema"` + Table string `yaml:"table" json:"table"` + OrgTable string `yaml:"org_table" json:"org_table"` + Name string `yaml:"name" json:"name"` + OrgName string `yaml:"org_name" json:"org_name"` + FixedLength byte `yaml:"fixed_length" json:"fixed_length"` + CharacterSet uint16 `yaml:"character_set" json:"character_set"` + ColumnLength uint32 `yaml:"column_length" json:"column_length"` + Type byte `yaml:"type" json:"type"` + Flags uint16 `yaml:"flags" json:"flags"` + Decimals byte `yaml:"decimals" json:"decimals"` + Filler []byte `yaml:"filler" json:"filler"` + DefaultValue string `yaml:"defaultValue" json:"defaultValue"` +} + +// Rows type TextRow struct { - Header Header `yaml:"header"` - Values []ColumnEntry `yaml:"values"` + Header Header `yaml:"header" json:"header"` + Values []ColumnEntry `yaml:"values" json:"values"` } type BinaryRow struct { - Header Header `yaml:"header"` - Values []ColumnEntry `yaml:"values"` - OkAfterRow bool `yaml:"okAfterRow"` - RowNullBuffer []byte `yaml:"rowNullBuffer"` + Header Header `yaml:"header" json:"header"` + Values []ColumnEntry `yaml:"values" json:"values"` + OkAfterRow bool `yaml:"okAfterRow" json:"okAfterRow"` + RowNullBuffer []byte `yaml:"rowNullBuffer" json:"rowNullBuffer"` } type ColumnEntry struct { - Type FieldType `yaml:"type"` - Name string `yaml:"name"` - Value interface{} `yaml:"value"` - Unsigned bool `yaml:"unsigned"` + Type FieldType `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` + Value interface{} `yaml:"value" json:"value"` + Unsigned bool `yaml:"unsigned" json:"unsigned"` } // COM_STMT_PREPARE packet type StmtPreparePacket struct { - Command byte `yaml:"command"` - Query string `yaml:"query"` + Command byte `yaml:"command" json:"command"` + Query string `yaml:"query" json:"query"` } // COM_STMT_PREPARE_OK packet type StmtPrepareOkPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - NumColumns uint16 `yaml:"num_columns"` - NumParams uint16 `yaml:"num_params"` - Filler byte `yaml:"filler"` - WarningCount uint16 `yaml:"warning_count"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + NumColumns uint16 `yaml:"num_columns" json:"num_columns"` + NumParams uint16 `yaml:"num_params" json:"num_params"` + Filler byte `yaml:"filler" json:"filler"` + WarningCount uint16 `yaml:"warning_count" json:"warning_count"` - ParamDefs []*ColumnDefinition41 `yaml:"param_definitions"` - EOFAfterParamDefs []byte `yaml:"eofAfterParamDefs"` - ColumnDefs []*ColumnDefinition41 `yaml:"column_definitions"` - EOFAfterColumnDefs []byte `yaml:"eofAfterColumnDefs"` + ParamDefs []*ColumnDefinition41 `yaml:"param_definitions" json:"param_definitions"` + EOFAfterParamDefs []byte `yaml:"eofAfterParamDefs" json:"eofAfterParamDefs"` + ColumnDefs []*ColumnDefinition41 `yaml:"column_definitions" json:"column_definitions"` + EOFAfterColumnDefs []byte `yaml:"eofAfterColumnDefs" json:"eofAfterColumnDefs"` } // COM_STMT_EXECUTE packet type StmtExecutePacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - Flags byte `yaml:"flags"` - IterationCount uint32 `yaml:"iteration_count"` - ParameterCount int `yaml:"parameter_count"` - NullBitmap []byte `yaml:"null_bitmap"` - NewParamsBindFlag byte `yaml:"new_params_bind_flag"` - Parameters []Parameter `yaml:"parameters"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + Flags byte `yaml:"flags" json:"flags"` + IterationCount uint32 `yaml:"iteration_count" json:"iteration_count"` + ParameterCount int `yaml:"parameter_count" json:"parameter_count"` + NullBitmap []byte `yaml:"null_bitmap" json:"null_bitmap"` + NewParamsBindFlag byte `yaml:"new_params_bind_flag" json:"new_params_bind_flag"` + Parameters []Parameter `yaml:"parameters" json:"parameters"` } type Parameter struct { - Type uint16 `yaml:"type"` - Unsigned bool `yaml:"unsigned"` - Name string `yaml:"name,omitempty"` - Value []byte `yaml:"value"` + Type uint16 `yaml:"type" json:"type"` + Unsigned bool `yaml:"unsigned" json:"unsigned"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Value []byte `yaml:"value" json:"value"` } // COM_STMT_FETCH packet is not currently supported because its response involves multi-resultset type StmtFetchPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - NumRows uint32 `yaml:"num_rows"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + NumRows uint32 `yaml:"num_rows" json:"num_rows"` } // COM_STMT_CLOSE packet type StmtClosePacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` } // COM_STMT_RESET packet type StmtResetPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` } // COM_STMT_SEND_LONG_DATA packet type StmtSendLongDataPacket struct { - Status byte `yaml:"status"` - StatementID uint32 `yaml:"statement_id"` - ParameterID uint16 `yaml:"parameter_id"` - Data []byte `yaml:"data"` + Status byte `yaml:"status" json:"status"` + StatementID uint32 `yaml:"statement_id" json:"statement_id"` + ParameterID uint16 `yaml:"parameter_id" json:"parameter_id"` + Data []byte `yaml:"data" json:"data"` } // Utility commands @@ -171,51 +171,51 @@ type StmtSendLongDataPacket struct { // COM_QUIT packet type QuitPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_INIT_DB packet type InitDBPacket struct { - Command byte `yaml:"command"` - Schema string `yaml:"schema"` + Command byte `yaml:"command" json:"command"` + Schema string `yaml:"schema" json:"schema"` } // COM_STATISTICS packet type StatisticsPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_DEBUG packet type DebugPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_PING packet type PingPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_RESET_CONNECTION packet type ResetConnectionPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` } // COM_SET_OPTION packet type SetOptionPacket struct { - Status byte `yaml:"status"` - Option uint16 `yaml:"option"` + Status byte `yaml:"status" json:"status"` + Option uint16 `yaml:"option" json:"option"` } // COM_CHANGE_USER packet (Not completed/supported as of now) //refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_change_user.html type ChangeUserPacket struct { - Command byte `yaml:"command"` + Command byte `yaml:"command" json:"command"` // rest of the fields are not present as the packet is not supported } diff --git a/pkg/models/mysql/conn.go b/pkg/models/mysql/conn.go index b7726e4b09..36bfe26989 100644 --- a/pkg/models/mysql/conn.go +++ b/pkg/models/mysql/conn.go @@ -7,55 +7,55 @@ package mysql // HandshakeV10Packet represents the initial handshake packet sent by the server to the client type HandshakeV10Packet struct { - ProtocolVersion uint8 `yaml:"protocol_version"` - ServerVersion string `yaml:"server_version"` - ConnectionID uint32 `yaml:"connection_id"` - AuthPluginData []byte `yaml:"auth_plugin_data,omitempty,flow"` - Filler byte `yaml:"filler"` - CapabilityFlags uint32 `yaml:"capability_flags"` - CharacterSet uint8 `yaml:"character_set"` - StatusFlags uint16 `yaml:"status_flags"` - AuthPluginName string `yaml:"auth_plugin_name"` + ProtocolVersion uint8 `yaml:"protocol_version" json:"protocol_version"` + ServerVersion string `yaml:"server_version" json:"server_version"` + ConnectionID uint32 `yaml:"connection_id" json:"connection_id"` + AuthPluginData []byte `yaml:"auth_plugin_data,omitempty,flow" json:"auth_plugin_data,omitempty"` + Filler byte `yaml:"filler" json:"filler"` + CapabilityFlags uint32 `yaml:"capability_flags" json:"capability_flags"` + CharacterSet uint8 `yaml:"character_set" json:"character_set"` + StatusFlags uint16 `yaml:"status_flags" json:"status_flags"` + AuthPluginName string `yaml:"auth_plugin_name" json:"auth_plugin_name"` } // HandshakeResponse41Packet represents the response packet sent by the client to the server after receiving the HandshakeV10Packet type HandshakeResponse41Packet struct { - CapabilityFlags uint32 `yaml:"capability_flags"` - MaxPacketSize uint32 `yaml:"max_packet_size"` - CharacterSet uint8 `yaml:"character_set"` - Filler [23]byte `yaml:"filler,omitempty,flow"` - Username string `yaml:"username"` - AuthResponse []byte `yaml:"auth_response,omitempty,flow"` - Database string `yaml:"database"` - AuthPluginName string `yaml:"auth_plugin_name"` - ConnectionAttributes map[string]string `yaml:"connection_attributes,omitempty"` - ZstdCompressionLevel byte `yaml:"zstdcompressionlevel"` + CapabilityFlags uint32 `yaml:"capability_flags" json:"capability_flags"` + MaxPacketSize uint32 `yaml:"max_packet_size" json:"max_packet_size"` + CharacterSet uint8 `yaml:"character_set" json:"character_set"` + Filler [23]byte `yaml:"filler,omitempty,flow" json:"filler,omitempty"` + Username string `yaml:"username" json:"username"` + AuthResponse []byte `yaml:"auth_response,omitempty,flow" json:"auth_response,omitempty"` + Database string `yaml:"database" json:"database"` + AuthPluginName string `yaml:"auth_plugin_name" json:"auth_plugin_name"` + ConnectionAttributes map[string]string `yaml:"connection_attributes,omitempty" json:"connection_attributes,omitempty"` + ZstdCompressionLevel byte `yaml:"zstdcompressionlevel" json:"zstdcompressionlevel"` } // Authentication Packets // AuthSwitchRequestPacket represents the packet sent by the server to the client to switch to a different authentication method type AuthSwitchRequestPacket struct { - StatusTag byte `yaml:"status_tag"` - PluginName string `yaml:"plugin_name"` - PluginData string `yaml:"plugin_data"` + StatusTag byte `yaml:"status_tag" json:"status_tag"` + PluginName string `yaml:"plugin_name" json:"plugin_name"` + PluginData string `yaml:"plugin_data" json:"plugin_data"` } // AuthSwitchResponsePacket represents the packet sent by the client to the server in response to an AuthSwitchRequestPacket. // Note: If the server sends an AuthMoreDataPacket, the client will continue sending AuthSwitchResponsePackets until the server sends an OK packet or an ERR packet. type AuthSwitchResponsePacket struct { - Data string `yaml:"data"` + Data string `yaml:"data" json:"data"` } // AuthMoreDataPacket represents the packet sent by the server to the client to request additional data for authentication type AuthMoreDataPacket struct { - StatusTag byte `yaml:"status_tag"` - Data string `yaml:"data"` + StatusTag byte `yaml:"status_tag" json:"status_tag"` + Data string `yaml:"data" json:"data"` } // AuthNextFactorPacket represents the packet sent by the server to the client to request the next factor for multi-factor authentication type AuthNextFactorPacket struct { - PacketType byte `yaml:"packet_type"` - PluginName string `yaml:"plugin_name"` - PluginData string `yaml:"plugin_data"` + PacketType byte `yaml:"packet_type" json:"packet_type"` + PluginName string `yaml:"plugin_name" json:"plugin_name"` + PluginData string `yaml:"plugin_data" json:"plugin_data"` } diff --git a/pkg/models/mysql/generic.go b/pkg/models/mysql/generic.go index 07be522951..358dea3b19 100644 --- a/pkg/models/mysql/generic.go +++ b/pkg/models/mysql/generic.go @@ -1,7 +1,7 @@ package mysql // This file contains structs for mysql generic response packets -//refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_response_packets.html +// refer: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_response_packets.html // OKPacket represents the OK packet sent by the server to the client, it represents a successful completion of a command type OKPacket struct { @@ -15,16 +15,16 @@ type OKPacket struct { // ERRPacket represents the ERR packet sent by the server to the client, it represents an error occurred during the execution of a command type ERRPacket struct { - Header byte `yaml:"header"` - ErrorCode uint16 `yaml:"error_code"` - SQLStateMarker string `yaml:"sql_state_marker"` - SQLState string `yaml:"sql_state"` - ErrorMessage string `yaml:"error_message"` + Header byte `json:"header" yaml:"header"` + ErrorCode uint16 `json:"error_code" yaml:"error_code"` + SQLStateMarker string `json:"sql_state_marker" yaml:"sql_state_marker"` + SQLState string `json:"sql_state" yaml:"sql_state"` + ErrorMessage string `json:"error_message" yaml:"error_message"` } // EOFPacket represents the EOF packet sent by the server to the client, it represents the end of a query execution result type EOFPacket struct { - Header byte `yaml:"header"` - Warnings uint16 `yaml:"warnings"` - StatusFlags uint16 `yaml:"status_flags"` + Header byte `json:"header" yaml:"header"` + Warnings uint16 `json:"warnings" yaml:"warnings"` + StatusFlags uint16 `json:"status_flags" yaml:"status_flags"` } diff --git a/pkg/models/mysql/mysql.go b/pkg/models/mysql/mysql.go index c1f6389a7e..3237134439 100644 --- a/pkg/models/mysql/mysql.go +++ b/pkg/models/mysql/mysql.go @@ -1,6 +1,9 @@ package mysql import ( + "encoding/json" + "errors" + "fmt" "time" "gopkg.in/yaml.v3" @@ -59,3 +62,303 @@ type Header struct { PayloadLength uint32 `json:"payload_length" yaml:"payload_length"` SequenceID uint8 `json:"sequence_id" yaml:"sequence_id"` } + +// custom marshal and unmarshal methods for Request and Response structs + +// MarshalJSON implements json.Marshaler for Request because of interface type of field 'Message' +func (r *Request) MarshalJSON() ([]byte, error) { + // create an alias struct to avoid infinite recursion + type RequestAlias struct { + Header *PacketInfo `json:"header"` + Message json.RawMessage `json:"message"` + Meta map[string]string `json:"meta,omitempty"` + } + + aux := RequestAlias{ + Header: r.Header, + Message: json.RawMessage(nil), + Meta: r.Meta, + } + + if r.Message != nil { + // Marshal the message interface{} into JSON + msgJSON, err := json.Marshal(r.Message) + if err != nil { + return nil, err + } + fmt.Println("msgJSON::", string(msgJSON)) + aux.Message = msgJSON + } + + // Marshal the alias struct into JSON + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler for Request because of interface type of field 'Message' +func (r *Request) UnmarshalJSON(data []byte) error { + // Alias struct to prevent recursion during unmarshalling + type RequestAlias struct { + Header *PacketInfo `json:"header"` + Message json.RawMessage `json:"message"` + Meta map[string]string `json:"meta,omitempty"` + } + var aux RequestAlias + + // Unmarshal the data into the alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Assign the unmarshalled data to the original struct + r.Header = aux.Header + r.Meta = aux.Meta + + // Unmarshal the message field based on the type in the header + switch r.Header.Type { + case HandshakeResponse41: + var msg HandshakeResponse41Packet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CachingSha2PasswordToString(RequestPublicKey): + var msg string + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = msg + + case "encrypted_password": + var msg string + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = msg + + case CommandStatusToString(COM_QUIT): + var msg QuitPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_INIT_DB): + var msg InitDBPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STATISTICS): + var msg StatisticsPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_DEBUG): + var msg DebugPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_PING): + var msg PingPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_CHANGE_USER): + var msg ChangeUserPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_RESET_CONNECTION): + var msg ResetConnectionPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_QUERY): + var msg QueryPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_PREPARE): + var msg StmtPreparePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_EXECUTE): + var msg StmtExecutePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_CLOSE): + var msg StmtClosePacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_RESET): + var msg StmtResetPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case CommandStatusToString(COM_STMT_SEND_LONG_DATA): + var msg StmtSendLongDataPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + default: + return errors.New("failed to unmarshal unknown request packet type") + } + + fmt.Println("r.Message::", r.Message) + + return nil +} + +// MarshalJSON implements json.Marshaler for Response because of interface type of field 'Message' +func (r *Response) MarshalJSON() ([]byte, error) { + // Alias to avoid recursion + type ResponseAlias struct { + PacketBundle `json:"packet_bundle"` + Payload string `json:"payload,omitempty"` + Message json.RawMessage `json:"message"` + } + + aux := ResponseAlias{ + PacketBundle: r.PacketBundle, + Payload: r.Payload, + } + + if r.Message != nil { + // Marshal the message interface{} into JSON + msgJSON, err := json.Marshal(r.Message) + if err != nil { + return nil, err + } + aux.Message = msgJSON + } + + return json.Marshal(aux) +} + +// UnmarshalJSON implements json.Unmarshaler for Response because of interface type of field 'Message' +func (r *Response) UnmarshalJSON(data []byte) error { + // Alias struct to prevent recursion + type ResponseAlias struct { + PacketBundle `json:"packet_bundle"` + Payload string `json:"payload,omitempty"` + Message json.RawMessage `json:"message"` + } + var aux ResponseAlias + + // Unmarshal the data into the alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + // Assign the unmarshalled data to the original struct + r.PacketBundle = aux.PacketBundle + r.Payload = aux.Payload + + // Unmarshal the message field based on the type in the header + switch r.PacketBundle.Header.Type { + // Generic response + case StatusToString(EOF): + var msg EOFPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case StatusToString(ERR): + var msg ERRPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case StatusToString(OK): + var msg OKPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + // Connection phase + case AuthStatusToString(HandshakeV10): + var msg HandshakeV10Packet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthSwitchRequest): + var msg AuthSwitchRequestPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthMoreData): + var msg AuthMoreDataPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case AuthStatusToString(AuthNextFactor): // not supported yet + var msg AuthNextFactorPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + // Command phase + case COM_STMT_PREPARE_OK: + var msg StmtPrepareOkPacket + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case string(Text): + var msg TextResultSet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + case string(Binary): + var msg BinaryProtocolResultSet + if err := json.Unmarshal(aux.Message, &msg); err != nil { + return err + } + r.Message = &msg + + default: + return errors.New("failed to unmarshal unknown response packet type") + } + + return nil +} diff --git a/pkg/models/postgres.go b/pkg/models/postgres.go index bb8306b076..faa4da6fdc 100755 --- a/pkg/models/postgres.go +++ b/pkg/models/postgres.go @@ -34,12 +34,12 @@ type Backend struct { CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` Describe pgproto3.Describe `json:"describe,omitempty" yaml:"describe,omitempty"` - Execute pgproto3.Execute `yaml:"-"` + Execute pgproto3.Execute `json:"-" yaml:"-"` Executes []pgproto3.Execute `json:"execute,omitempty" yaml:"execute,omitempty"` Flush pgproto3.Flush `json:"flush,omitempty" yaml:"flush,omitempty"` FunctionCall pgproto3.FunctionCall `json:"function_call,omitempty" yaml:"function_call,omitempty"` GssEncRequest pgproto3.GSSEncRequest `json:"gss_enc_request,omitempty" yaml:"gss_enc_request,omitempty"` - Parse pgproto3.Parse `yaml:"-"` + Parse pgproto3.Parse `json:"-" yaml:"-"` Parses []pgproto3.Parse `json:"parse,omitempty" yaml:"parse,omitempty"` Query pgproto3.Query `json:"query,omitempty" yaml:"query,omitempty"` SSlRequest pgproto3.SSLRequest `json:"ssl_request,omitempty" yaml:"ssl_request,omitempty"` @@ -70,45 +70,33 @@ type Frontend struct { AuthenticationSASLContinue pgproto3.AuthenticationSASLContinue `json:"authentication_sasl_continue,omitempty" yaml:"authentication_sasl_continue,omitempty,flow"` AuthenticationSASLFinal pgproto3.AuthenticationSASLFinal `json:"authentication_sasl_final,omitempty" yaml:"authentication_sasl_final,omitempty,flow"` BackendKeyData pgproto3.BackendKeyData `json:"backend_key_data,omitempty" yaml:"backend_key_data,omitempty"` - BindComplete pgproto3.BindComplete `yaml:"-"` + BindComplete pgproto3.BindComplete `json:"-" yaml:"-"` BindCompletes []pgproto3.BindComplete `json:"bind_complete,omitempty" yaml:"bind_complete,omitempty"` CloseComplete pgproto3.CloseComplete `json:"close_complete,omitempty" yaml:"close_complete,omitempty"` - CommandComplete pgproto3.CommandComplete `yaml:"-"` + CommandComplete pgproto3.CommandComplete `json:"-" yaml:"-"` CommandCompletes []pgproto3.CommandComplete `json:"command_complete,omitempty" yaml:"command_complete,omitempty"` - CopyBothResponse pgproto3.CopyBothResponse `json:"copy_both_response,omitempty" yaml:"copy_both_response,omitempty"` - CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` - CopyInResponse pgproto3.CopyInResponse `json:"copy_in_response,omitempty" yaml:"copy_in_response,omitempty"` - CopyOutResponse pgproto3.CopyOutResponse `json:"copy_out_response,omitempty" yaml:"copy_out_response,omitempty"` - CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` - DataRow pgproto3.DataRow `yaml:"-"` - DataRows []pgproto3.DataRow `json:"data_row,omitempty" yaml:"data_row,omitempty,flow"` - EmptyQueryResponse pgproto3.EmptyQueryResponse `json:"empty_query_response,omitempty" yaml:"empty_query_response,omitempty"` - ErrorResponse pgproto3.ErrorResponse `json:"error_response,omitempty" yaml:"error_response,omitempty"` - FunctionCallResponse pgproto3.FunctionCallResponse `json:"function_call_response,omitempty" yaml:"function_call_response,omitempty"` - NoData pgproto3.NoData `json:"no_data,omitempty" yaml:"no_data,omitempty"` - NoticeResponse pgproto3.NoticeResponse `json:"notice_response,omitempty" yaml:"notice_response,omitempty"` - NotificationResponse pgproto3.NotificationResponse `json:"notification_response,omitempty" yaml:"notification_response,omitempty"` - ParameterDescription pgproto3.ParameterDescription `json:"parameter_description,omitempty" yaml:"parameter_description,omitempty"` - ParameterStatus pgproto3.ParameterStatus `yaml:"-"` - ParameterStatusCombined []pgproto3.ParameterStatus `json:"parameter_status,omitempty" yaml:"parameter_status,omitempty"` - ParseComplete pgproto3.ParseComplete `yaml:"-"` - ParseCompletes []pgproto3.ParseComplete `json:"parse_complete,omitempty" yaml:"parse_complete,omitempty"` - ReadyForQuery pgproto3.ReadyForQuery `json:"ready_for_query,omitempty" yaml:"ready_for_query,omitempty"` - RowDescription pgproto3.RowDescription `json:"row_description,omitempty" yaml:"row_description,omitempty,flow"` - PortalSuspended pgproto3.PortalSuspended `json:"portal_suspended,omitempty" yaml:"portal_suspended,omitempty"` - MsgType byte `json:"msg_type,omitempty" yaml:"msg_type,omitempty"` - AuthType int32 `json:"auth_type" yaml:"auth_type"` - // AuthMechanism string `json:"auth_mechanism,omitempty" yaml:"auth_mechanism,omitempty"` - BodyLen int `json:"body_len,omitempty" yaml:"body_len,omitempty"` -} - -type StartupPacket struct { - Length uint32 - ProtocolVersion uint32 -} - -type RegularPacket struct { - Identifier byte - Length uint32 - Payload []byte + // CopyBothResponse pgproto3.CopyBothResponse `json:"copy_both_response,omitempty" yaml:"copy_both_response,omitempty"` + CopyData pgproto3.CopyData `json:"copy_data,omitempty" yaml:"copy_data,omitempty"` + // CopyInResponse pgproto3.CopyInResponse `json:"copy_in_response,omitempty" yaml:"copy_in_response,omitempty"` + // CopyOutResponse pgproto3.CopyOutResponse `json:"copy_out_response,omitempty" yaml:"copy_out_response,omitempty"` + CopyDone pgproto3.CopyDone `json:"copy_done,omitempty" yaml:"copy_done,omitempty"` + DataRow pgproto3.DataRow `json:"-" yaml:"-"` + DataRows []pgproto3.DataRow `json:"data_row,omitempty" yaml:"data_row,omitempty,flow"` + EmptyQueryResponse pgproto3.EmptyQueryResponse `json:"empty_query_response,omitempty" yaml:"empty_query_response,omitempty"` + ErrorResponse pgproto3.ErrorResponse `json:"error_response,omitempty" yaml:"error_response,omitempty"` + FunctionCallResponse pgproto3.FunctionCallResponse `json:"function_call_response,omitempty" yaml:"function_call_response,omitempty"` + NoData pgproto3.NoData `json:"no_data,omitempty" yaml:"no_data,omitempty"` + NoticeResponse pgproto3.NoticeResponse `json:"notice_response,omitempty" yaml:"notice_response,omitempty"` + NotificationResponse pgproto3.NotificationResponse `json:"notification_response,omitempty" yaml:"notification_response,omitempty"` + ParameterDescription pgproto3.ParameterDescription `json:"parameter_description,omitempty" yaml:"parameter_description,omitempty"` + ParameterStatus pgproto3.ParameterStatus `json:"-" yaml:"-"` + ParameterStatusCombined []pgproto3.ParameterStatus `json:"parameter_status,omitempty" yaml:"parameter_status,omitempty"` + ParseComplete pgproto3.ParseComplete `json:"-" yaml:"-"` + ParseCompletes []pgproto3.ParseComplete `json:"parse_complete,omitempty" yaml:"parse_complete,omitempty"` + ReadyForQuery pgproto3.ReadyForQuery `json:"ready_for_query,omitempty" yaml:"ready_for_query,omitempty"` + RowDescription pgproto3.RowDescription `json:"row_description,omitempty" yaml:"row_description,omitempty,flow"` + PortalSuspended pgproto3.PortalSuspended `json:"portal_suspended,omitempty" yaml:"portal_suspended,omitempty"` + MsgType byte `json:"msg_type,omitempty" yaml:"msg_type,omitempty"` + AuthType int32 `json:"auth_type" yaml:"auth_type"` + BodyLen int `json:"body_len,omitempty" yaml:"body_len,omitempty"` } diff --git a/pkg/models/ut.go b/pkg/models/ut.go index 22eed68dff..a6286aff86 100644 --- a/pkg/models/ut.go +++ b/pkg/models/ut.go @@ -18,11 +18,12 @@ type UTDetails struct { } type UT struct { - TestBehavior string `yaml:"test_behavior"` - TestName string `yaml:"test_name"` - TestCode string `yaml:"test_code"` - NewImportsCode string `yaml:"new_imports_code"` - TestsTags string `yaml:"tests_tags"` + TestBehavior string `yaml:"test_behavior"` + TestName string `yaml:"test_name"` + TestCode string `yaml:"test_code"` + NewImportsCode string `yaml:"new_imports_code"` + LibraryInstallationCode string `yaml:"library_installation_code"` + TestsTags string `yaml:"tests_tags"` } type FailedUT struct { diff --git a/pkg/platform/docker/docker.go b/pkg/platform/docker/docker.go index 56b7d7436b..1f1b9cda14 100644 --- a/pkg/platform/docker/docker.go +++ b/pkg/platform/docker/docker.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types" dockerContainerPkg "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" @@ -154,7 +153,7 @@ func (idc *Impl) StopAndRemoveDockerContainer() error { } } - removeOptions := types.ContainerRemoveOptions{ + removeOptions := dockerContainerPkg.RemoveOptions{ RemoveVolumes: true, Force: true, } @@ -175,7 +174,7 @@ func (idc *Impl) NetworkExists(networkName string) (bool, error) { defer cancel() // Retrieve all networks. - networks, err := idc.NetworkList(ctx, types.NetworkListOptions{}) + networks, err := idc.NetworkList(ctx, network.ListOptions{}) if err != nil { return false, fmt.Errorf("error retrieving networks: %v", err) } @@ -195,7 +194,7 @@ func (idc *Impl) CreateNetwork(networkName string) error { ctx, cancel := context.WithTimeout(context.Background(), idc.timeoutForDockerQuery) defer cancel() - _, err := idc.NetworkCreate(ctx, networkName, types.NetworkCreate{ + _, err := idc.NetworkCreate(ctx, networkName, network.CreateOptions{ Driver: "bridge", }) @@ -360,7 +359,7 @@ func (idc *Impl) GetHostWorkingDirectory() (string, error) { // Loop through container mounts and find the mount for current directory in the container for _, mount := range containerMounts { if mount.Destination == curDir { - idc.logger.Debug(fmt.Sprintf("found mount for %s in keploy-v2 container", curDir), zap.Any("mount", mount)) + idc.logger.Info(fmt.Sprintf("found mount for %s in keploy-v2 container", curDir), zap.Any("mount", mount)) return mount.Source, nil } } @@ -521,6 +520,30 @@ func (idc *Impl) SetKeployNetwork(c *Compose) (*NetworkInfo, error) { return networkInfo, nil } +func (idc *Impl) SetInitPid(c *Compose) error { + // Add or modify network for each service + for _, service := range c.Services.Content { + pidFound := false + for _, item := range service.Content { + if item.Value == "pid" { + pidFound = true + break + } + } + + if !pidFound { + service.Content = append(service.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "pid"}, + &yaml.Node{ + Kind: yaml.ScalarNode, + Value: "container:keploy-init", + }, + ) + } + } + return nil +} + // IsContainerRunning check if the container is already running or not, required for docker start command. func (idc *Impl) IsContainerRunning(containerName string) (bool, error) { diff --git a/pkg/platform/docker/service.go b/pkg/platform/docker/service.go index d41dea9263..cb20878205 100644 --- a/pkg/platform/docker/service.go +++ b/pkg/platform/docker/service.go @@ -27,7 +27,7 @@ type Client interface { SetKeployNetwork(c *Compose) (*NetworkInfo, error) ReadComposeFile(filePath string) (*Compose, error) WriteComposeFile(compose *Compose, path string) error - + SetInitPid(c *Compose) error IsContainerRunning(containerName string) (bool, error) CreateVolume(ctx context.Context, volumeName string, recreate bool) error } diff --git a/pkg/platform/docker/util.go b/pkg/platform/docker/util.go index f35f46ef73..f66e1afd2f 100644 --- a/pkg/platform/docker/util.go +++ b/pkg/platform/docker/util.go @@ -1,14 +1,34 @@ -//go:build linux +//go:build !windows package docker import ( + "context" + "errors" "fmt" + "os" + "os/exec" "regexp" + "runtime" + "strings" + "syscall" + "github.com/docker/docker/api/types/network" + "go.keploy.io/server/v2/config" "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/term" ) +type ConfigStruct struct { + DockerImage string + Envs map[string]string +} + +var DockerConfig = ConfigStruct{ + DockerImage: "ghcr.io/keploy/keploy", +} + func ParseDockerCmd(cmd string, kind utils.CmdType, idc Client) (string, string, error) { // Regular expression patterns @@ -51,3 +71,258 @@ func ParseDockerCmd(cmd string, kind utils.CmdType, idc Client) (string, string, return containerName, networkName, nil } + +func GenerateDockerEnvs(config ConfigStruct) string { + var envs []string + for key, value := range config.Envs { + envs = append(envs, fmt.Sprintf("-e %s='%s'", key, value)) + } + return strings.Join(envs, " ") +} + +// StartInDocker will check if the docker command is provided as an input +// then start the Keploy as a docker container and run the command +// should also return a boolean if the execution is moved to docker +func StartInDocker(ctx context.Context, logger *zap.Logger, conf *config.Config) error { + + if DockerConfig.Envs == nil { + DockerConfig.Envs = map[string]string{ + "INSTALLATION_ID": conf.InstallationID, + } + } else { + DockerConfig.Envs["INSTALLATION_ID"] = conf.InstallationID + } + + //Check if app command starts with docker or docker-compose. + // If it does, then we would run the docker version of keploy and + + // pass the all the commands and args to the docker version of Keploy + + err := RunInDocker(ctx, logger) + if err != nil { + utils.LogError(logger, err, "failed to run the command in docker") + return err + } + // gracefully exit the current process + logger.Info("exiting the current process as the command is moved to docker") + os.Exit(0) + return nil +} + +func RunInDocker(ctx context.Context, logger *zap.Logger) error { + //Get the correct keploy alias. + keployAlias, err := getAlias(ctx, logger) + if err != nil { + return err + } + fmt.Println("Running in docker keploy alias", keployAlias) + + client, err := New(logger) + if err != nil { + utils.LogError(logger, err, "failed to initalise docker") + return err + } + + addKeployNetwork(ctx, logger, client) + err = client.CreateVolume(ctx, "debugfs", true) + if err != nil { + utils.LogError(logger, err, "failed to debugfs volume") + return err + } + + var cmd *exec.Cmd + + // Detect the operating system + if runtime.GOOS == "windows" { + var args []string + args = append(args, "/C") + args = append(args, strings.Split(keployAlias, " ")...) + args = append(args, os.Args[1:]...) + // Use cmd.exe /C for Windows + cmd = exec.CommandContext( + ctx, + "cmd.exe", + args..., + ) + } else { + // Use sh -c for Unix-like systems + cmd = exec.CommandContext( + ctx, + "sh", + "-c", + keployAlias, + ) + } + + cmd.Cancel = func() error { + err := utils.SendSignal(logger, -cmd.Process.Pid, syscall.SIGINT) + if err != nil { + utils.LogError(logger, err, "failed to start stop docker") + return err + } + return nil + } + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logger.Info("running the following command in docker", zap.String("command", cmd.String())) + err = cmd.Run() + if err != nil { + if ctx.Err() == context.Canceled { + return ctx.Err() + } + utils.LogError(logger, err, "failed to start keploy in docker") + return err + } + return nil +} + +func getAlias(ctx context.Context, logger *zap.Logger) (string, error) { + // Get the name of the operating system. + osName := runtime.GOOS + //TODO: configure the hardcoded port mapping + img := DockerConfig.DockerImage + ":v" + utils.Version + logger.Info("Starting keploy in docker with image", zap.String("image:", img)) + envs := GenerateDockerEnvs(DockerConfig) + if envs != "" { + envs = envs + " " + } + var ttyFlag string + + if term.IsTerminal(int(os.Stdin.Fd())) { + // ttyFlag = " -it " + ttyFlag = " " + } else { + ttyFlag = " " + } + + switch osName { + case "linux": + alias := "sudo docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + " -v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + case "windows": + // Get the current working directory + pwd, err := os.Getwd() + if err != nil { + utils.LogError(logger, err, "failed to get the current working directory") + } + dpwd := convertPathToUnixStyle(pwd) + cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") + out, err := cmd.Output() + if err != nil { + utils.LogError(logger, err, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] + if len(dockerContext) == 0 { + utils.LogError(logger, nil, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext = strings.Split(dockerContext, "\n")[0] + if dockerContext == "colima" { + logger.Info("Starting keploy in docker with colima context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img + return alias, nil + } + // if default docker context is used + logger.Info("Starting keploy in docker with default context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + pwd + ":" + dpwd + " -w " + dpwd + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("USERPROFILE") + "\\.keploy-config:/root/.keploy-config -v " + os.Getenv("USERPROFILE") + "\\.keploy:/root/.keploy --rm " + img + return alias, nil + case "darwin": + cmd := exec.CommandContext(ctx, "docker", "context", "ls", "--format", "{{.Name}}\t{{.Current}}") + out, err := cmd.Output() + if err != nil { + utils.LogError(logger, err, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext := strings.Split(strings.TrimSpace(string(out)), "\n")[0] + if len(dockerContext) == 0 { + utils.LogError(logger, nil, "failed to get the current docker context") + return "", errors.New("failed to get alias") + } + dockerContext = strings.Split(dockerContext, "\n")[0] + if dockerContext == "colima" { + logger.Info("Starting keploy in docker with colima context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + } + // if default docker context is used + logger.Info("Starting keploy in docker with default context, as that is the current context.") + alias := "docker container run --name keploy-v2 " + envs + "-e BINARY_TO_DOCKER=true -p 36789:36789 -p 8096:8096 --privileged --pid=host" + ttyFlag + "-v " + os.Getenv("PWD") + ":" + os.Getenv("PWD") + " -w " + os.Getenv("PWD") + " -v /sys/fs/cgroup:/sys/fs/cgroup -v debugfs:/sys/kernel/debug:rw -v /sys/fs/bpf:/sys/fs/bpf -v /var/run/docker.sock:/var/run/docker.sock -v " + os.Getenv("HOME") + "/.keploy-config:/root/.keploy-config -v " + os.Getenv("HOME") + "/.keploy:/root/.keploy --rm " + img + return alias, nil + } + return "", errors.New("failed to get alias") +} + +func addKeployNetwork(ctx context.Context, logger *zap.Logger, client Client) { + + // Check if the 'keploy-network' network exists + networks, err := client.NetworkList(ctx, network.ListOptions{}) + if err != nil { + logger.Debug("failed to list docker networks") + return + } + + for _, network := range networks { + if network.Name == "keploy-network" { + logger.Debug("keploy network already exists") + return + } + } + + // Create the 'keploy' network if it doesn't exist + _, err = client.NetworkCreate(ctx, "keploy-network", network.CreateOptions{}) + if err != nil { + logger.Debug("failed to create keploy network") + return + } + + logger.Debug("keploy network created") +} + +func convertPathToUnixStyle(path string) string { + // Replace backslashes with forward slashes + unixPath := strings.Replace(path, "\\", "/", -1) + // Remove 'C:' + if len(unixPath) > 1 && unixPath[1] == ':' { + unixPath = unixPath[2:] + } + return unixPath +} + +// ExtractPidNamespaceInode extracts the inode of the PID namespace of a given PID +func ExtractPidNamespaceInode(pid int) (string, error) { + // Check the OS + if runtime.GOOS != "linux" { + // Execute command in the container to get the PID namespace + output, err := exec.Command("docker", "exec", "keploy-init", "stat", "/proc/1/ns/pid").Output() + if err != nil { + return "", err + } + outputStr := string(output) + + // Use a regular expression to extract the inode from the output + re := regexp.MustCompile(`pid:\[(\d+)\]`) + match := re.FindStringSubmatch(outputStr) + + if len(match) < 2 { + return "", fmt.Errorf("failed to extract PID namespace inode") + } + + pidNamespace := match[1] + fmt.Println("SENDING PID namespace inode:", pidNamespace) + return pidNamespace, nil + } + + // Check the namespace file in /proc + nsPath := fmt.Sprintf("/proc/%d/ns/pid", pid) + fileInfo, err := os.Stat(nsPath) + if err != nil { + return "", err + } + + // Retrieve inode number + inode := fileInfo.Sys().(*syscall.Stat_t).Ino + return fmt.Sprintf("%d", inode), nil +} diff --git a/pkg/platform/http/agent.go b/pkg/platform/http/agent.go new file mode 100644 index 0000000000..2674c00690 --- /dev/null +++ b/pkg/platform/http/agent.go @@ -0,0 +1,705 @@ +//go:build !windows + +package http + +import ( + "bytes" + "context" + _ "embed" // necessary for embedding + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "sync" + "syscall" + "time" + + "github.com/docker/docker/api/types/events" + "go.keploy.io/server/v2/config" + "go.keploy.io/server/v2/pkg/core/app" + "go.keploy.io/server/v2/pkg/core/hooks" + "go.keploy.io/server/v2/pkg/models" + kdocker "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type AgentClient struct { + logger *zap.Logger + dockerClient kdocker.Client //embedding the docker client to transfer the docker client methods to the core object + apps sync.Map + client http.Client + conf *config.Config +} + +//go:embed assets/initStop.sh +var initStopScript []byte + +func New(logger *zap.Logger, client kdocker.Client, c *config.Config) *AgentClient { + + return &AgentClient{ + logger: logger, + dockerClient: client, + client: http.Client{}, + conf: c, + } +} + +func (a *AgentClient) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + requestBody := models.IncomingReq{ + IncomingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for incoming request") + return nil, fmt.Errorf("error marshaling request body for incoming request: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/incoming", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for incoming request") + return nil, fmt.Errorf("error creating request for incoming request: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get incoming: %s", err.Error()) + } + + // Ensure response body is closed when we're done + go func() { + <-ctx.Done() + if res.Body != nil { + _ = res.Body.Close() + } + }() + + // Create a channel to stream TestCase data + tcChan := make(chan *models.TestCase) + + go func() { + defer close(tcChan) + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for incoming request") + } + }() + + decoder := json.NewDecoder(res.Body) + fmt.Println("Starting to read from the response body") + + for { + var testCase models.TestCase + if err := decoder.Decode(&testCase); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + // End of the stream + break + } + utils.LogError(a.logger, err, "failed to decode test case from stream") + break + } + + select { + case <-ctx.Done(): + // If the context is done, exit the loop + return + case tcChan <- &testCase: + // Send the decoded test case to the channel + } + } + }() + + return tcChan, nil +} + +func (a *AgentClient) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { + requestBody := models.OutgoingReq{ + OutgoingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for mock outgoing") + return nil, fmt.Errorf("error marshaling request body for mock outgoing: %s", err.Error()) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/outgoing", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for mock outgoing") + return nil, fmt.Errorf("error creating request for mock outgoing: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to authenticate: %s", err.Error()) + } + + // Create a channel to stream Mock data + mockChan := make(chan *models.Mock) + + go func() { + defer close(mockChan) + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for mock outgoing") + } + }() + + decoder := json.NewDecoder(res.Body) + + // Read from the response body as a stream + for { + var mock models.Mock + if err := decoder.Decode(&mock); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + // End of the stream + break + } + utils.LogError(a.logger, err, "failed to decode mock from stream") + // break, it will exit the loop if there is any decoding error from the stream + } + + select { + case <-ctx.Done(): + // If the context is done, exit the loop + return + case mockChan <- &mock: + } + } + }() + + return mockChan, nil +} + +func (a *AgentClient) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { + // make a request to the server to mock outgoing + requestBody := models.OutgoingReq{ + OutgoingOptions: opts, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for mock outgoing") + return fmt.Errorf("error marshaling request body for mock outgoing: %s", err.Error()) + } + + // mock outgoing request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/mock", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for mock outgoing") + return fmt.Errorf("error creating request for mock outgoing: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request for mockOutgoing: %s", err.Error()) + } + + var mockResp models.AgentResp + err = json.NewDecoder(res.Body).Decode(&mockResp) + if err != nil { + return fmt.Errorf("failed to decode response body for mock outgoing: %s", err.Error()) + } + + if mockResp.Error != nil { + return mockResp.Error + } + + return nil + +} + +func (a *AgentClient) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { + requestBody := models.SetMocksReq{ + Filtered: filtered, + UnFiltered: unFiltered, + ClientID: id, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for setmocks") + return fmt.Errorf("error marshaling request body for setmocks: %s", err.Error()) + } + + // mock outgoing request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("http://localhost:%d/agent/setmocks", a.conf.Agent.Port), bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to create request for setmocks outgoing") + return fmt.Errorf("error creating request for set mocks: %s", err.Error()) + } + req.Header.Set("Content-Type", "application/json") + + // Make the HTTP request + res, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request for setmocks: %s", err.Error()) + } + + var mockResp models.AgentResp + err = json.NewDecoder(res.Body).Decode(&mockResp) + if err != nil { + return fmt.Errorf("failed to decode response body for setmocks: %s", err.Error()) + } + + if mockResp.Error != nil { + return mockResp.Error + } + + return nil +} + +func (a *AgentClient) GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) { + // Create the URL with query parameters + url := fmt.Sprintf("http://localhost:%d/agent/consumedmocks?id=%d", a.conf.Agent.Port, id) + + // Create a new GET request with the query parameter + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %s", err.Error()) + } + + req.Header.Set("Content-Type", "application/json") + + res, err := a.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request for mockOutgoing: %s", err.Error()) + } + + defer func() { + err := res.Body.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close response body for getconsumedmocks") + } + }() + + var consumedMocks []string + err = json.NewDecoder(res.Body).Decode(&consumedMocks) + if err != nil { + return nil, fmt.Errorf("failed to decode response body: %s", err.Error()) + } + + return consumedMocks, nil +} + +func (a *AgentClient) UnHook(_ context.Context, _ uint64) error { + return nil +} + +func (a *AgentClient) GetContainerIP(_ context.Context, id uint64) (string, error) { + + app, err := a.getApp(id) + if err != nil { + utils.LogError(a.logger, err, "failed to get app") + return "", err + } + + ip := app.ContainerIPv4Addr() + a.logger.Debug("ip address of the target app container", zap.Any("ip", ip)) + if ip == "" { + return "", fmt.Errorf("failed to get the IP address of the app container. Try increasing --delay (in seconds)") + } + + return ip, nil +} + +func (a *AgentClient) Run(ctx context.Context, id uint64, _ models.RunOptions) models.AppError { + + app, err := a.getApp(id) + if err != nil { + utils.LogError(a.logger, err, "failed to get app while running") + return models.AppError{AppErrorType: models.ErrInternal, Err: err} + } + + runAppErrGrp, runAppCtx := errgroup.WithContext(ctx) + + appErrCh := make(chan models.AppError, 1) + + defer func() { + err := runAppErrGrp.Wait() + + if err != nil { + utils.LogError(a.logger, err, "failed to stop the app") + } + }() + + runAppErrGrp.Go(func() error { + defer utils.Recover(a.logger) + defer close(appErrCh) + appErr := app.Run(runAppCtx) + if appErr.Err != nil { + utils.LogError(a.logger, appErr.Err, "error while running the app") + appErrCh <- appErr + } + return nil + }) + + select { + case <-runAppCtx.Done(): + fmt.Println("Context is canceled in the run app function") + return models.AppError{AppErrorType: models.ErrCtxCanceled, Err: nil} + case appErr := <-appErrCh: + return appErr + } +} + +func (a *AgentClient) Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) { + + // clientID := utils.GenerateID() + var clientID uint64 + + isDockerCmd := utils.IsDockerCmd(utils.CmdType(opts.CommandType)) + + // check if the agent is running + isAgentRunning := a.isAgentRunning(ctx) + + if !isAgentRunning { + // Start the keploy agent as a detached process and pipe the logs into a file + if !isDockerCmd && runtime.GOOS != "linux" { + return 0, fmt.Errorf("Operating system not supported for this feature") + } + if isDockerCmd { + // run the docker container instead of the agent binary + go func() { + if err := a.StartInDocker(ctx, a.logger); err != nil { + a.logger.Error("failed to start docker agent", zap.Error(err)) + } + }() + } else { + // Open the log file in append mode or create it if it doesn't exist + logFile, err := os.OpenFile("keploy_agent.log", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + utils.LogError(a.logger, err, "failed to open log file") + return 0, err + } + + defer func() { + err := logFile.Close() + if err != nil { + utils.LogError(a.logger, err, "failed to close agent log file") + } + }() + agentCmd := exec.Command("sudo", "keployv2", "agent") + agentCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} // Detach the process + + // Redirect the standard output and error to the log file + agentCmd.Stdout = logFile + agentCmd.Stderr = logFile + + err = agentCmd.Start() + if err != nil { + utils.LogError(a.logger, err, "failed to start keploy agent") + return 0, err + } + + a.logger.Info("keploy agent started", zap.Int("pid", agentCmd.Process.Pid)) + } + } + + runningChan := make(chan bool) + + // Start a goroutine to check if the agent is running + go func() { + for { + if a.isAgentRunning(ctx) { + // Signal that the agent is running + runningChan <- true + return + } + time.Sleep(1 * time.Second) // Poll every second + } + }() + + // inode := make(chan uint64) + var inode uint64 + if <-runningChan { + // check if its docker then create a init container first + // and then the app container + usrApp := app.NewApp(a.logger, clientID, cmd, a.dockerClient, app.Options{ + DockerNetwork: opts.DockerNetwork, + Container: opts.Container, + DockerDelay: opts.DockerDelay, + }) + a.apps.Store(clientID, usrApp) + + err := usrApp.Setup(ctx) + if err != nil { + utils.LogError(a.logger, err, "failed to setup app") + return 0, err + } + + if isDockerCmd { + // Start the init container to get the pid namespace + inode, err = a.Initcontainer(ctx, app.Options{ + DockerNetwork: usrApp.KeployNetwork, + Container: opts.Container, + DockerDelay: opts.DockerDelay, + }) + if err != nil { + utils.LogError(a.logger, err, "failed to setup init container") + } + } + opts.ClientID = clientID + opts.AppInode = inode + // Register the client with the server + err = a.RegisterClient(ctx, opts) + if err != nil { + utils.LogError(a.logger, err, "failed to register client") + return 0, err + } + } + + isAgentRunning = a.isAgentRunning(ctx) + if !isAgentRunning { + return 0, fmt.Errorf("keploy agent is not running, please start the agent first") + } + + return clientID, nil +} + +func (a *AgentClient) getApp(id uint64) (*app.App, error) { + ap, ok := a.apps.Load(id) + if !ok { + return nil, fmt.Errorf("app with id:%v not found", id) + } + + // type assertion on the app + h, ok := ap.(*app.App) + if !ok { + return nil, fmt.Errorf("failed to type assert app with id:%v", id) + } + + return h, nil +} + +// RegisterClient registers the client with the server +func (a *AgentClient) RegisterClient(ctx context.Context, opts models.SetupOptions) error { + + isAgent := a.isAgentRunning(ctx) + if !isAgent { + return fmt.Errorf("keploy agent is not running, please start the agent first") + } + + // Register the client with the server + clientPid := uint32(os.Getpid()) + + // start the app container and get the inode number + // keploy agent would have already runnning, + var inode uint64 + var err error + if runtime.GOOS == "linux" { + // send the network info to the kernel + inode, err = hooks.GetSelfInodeNumber() + if err != nil { + a.logger.Error("failed to get inode number") + } + } + + // Register the client with the server + requestBody := models.RegisterReq{ + SetupOptions: models.SetupOptions{ + DockerNetwork: opts.DockerNetwork, + ClientNsPid: clientPid, + Mode: opts.Mode, + ClientID: 0, + ClientInode: inode, + IsDocker: a.conf.Agent.IsDocker, + AppInode: opts.AppInode, + }, + } + + requestJSON, err := json.Marshal(requestBody) + if err != nil { + utils.LogError(a.logger, err, "failed to marshal request body for register client") + return fmt.Errorf("error marshaling request body for register client: %s", err.Error()) + } + + resp, err := a.client.Post(fmt.Sprintf("http://localhost:%d/agent/register", a.conf.Agent.Port), "application/json", bytes.NewBuffer(requestJSON)) + if err != nil { + utils.LogError(a.logger, err, "failed to send register client request") + return fmt.Errorf("error sending register client request: %s", err.Error()) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to register client: %s", resp.Status) + } + + // TODO: Read the response body in which we return the app id + var RegisterResp models.AgentResp + err = json.NewDecoder(resp.Body).Decode(&RegisterResp) + if err != nil { + utils.LogError(a.logger, err, "failed to decode response body for register client") + return fmt.Errorf("error decoding response body for register client: %s", err.Error()) + } + + if RegisterResp.Error != nil { + return RegisterResp.Error + } + + return nil +} + +func (a *AgentClient) StartInDocker(ctx context.Context, logger *zap.Logger) error { + + // Start the keploy agent in a Docker container + agentCtx := context.WithoutCancel(ctx) + + err := kdocker.StartInDocker(agentCtx, logger, &config.Config{ + InstallationID: a.conf.InstallationID, + }) + if err != nil { + utils.LogError(logger, err, "failed to start keploy agent in docker") + return err + } + return nil +} + +func (a *AgentClient) Initcontainer(ctx context.Context, opts app.Options) (uint64, error) { + // Create a temporary file for the embedded initStop.sh script + initFile, err := os.CreateTemp("", "initStop.sh") + if err != nil { + a.logger.Error("failed to create temporary file", zap.Error(err)) + return 0, err + } + defer func() { + err := os.Remove(initFile.Name()) + if err != nil { + a.logger.Error("failed to remove temporary file", zap.Error(err)) + } + }() + + _, err = initFile.Write(initStopScript) + if err != nil { + a.logger.Error("failed to write script to temporary file", zap.Error(err)) + return 0, err + } + + // Close the file after writing to avoid 'text file busy' error + if err := initFile.Close(); err != nil { + a.logger.Error("failed to close temporary file", zap.Error(err)) + return 0, err + } + + err = os.Chmod(initFile.Name(), 0755) + if err != nil { + a.logger.Error("failed to make temporary script executable", zap.Error(err)) + return 0, err + } + + // Create a channel to signal when the container starts + containerStarted := make(chan struct{}) + + // Start the Docker events listener in a separate goroutine + go func() { + events, errs := a.dockerClient.Events(ctx, events.ListOptions{}) + for { + select { + case event := <-events: + if event.Type == "container" && event.Action == "start" && event.Actor.Attributes["name"] == "keploy-init" { + a.logger.Info("Container keploy-init started") + containerStarted <- struct{}{} + return + } + case err := <-errs: + a.logger.Error("Error while listening to Docker events", zap.Error(err)) + return + case <-ctx.Done(): + return + } + } + }() + + // Start the init container to get the PID namespace inode + cmdCancel := func(cmd *exec.Cmd) func() error { + return func() error { + a.logger.Info("sending SIGINT to the Initcontainer", zap.Any("cmd.Process.Pid", cmd.Process.Pid)) + err := utils.SendSignal(a.logger, -cmd.Process.Pid, syscall.SIGINT) + return err + } + } + + cmd := fmt.Sprintf("docker run --network=%s --name keploy-init --rm -v%s:/initStop.sh alpine /initStop.sh", opts.DockerNetwork, initFile.Name()) + + // Execute the command + grp, ok := ctx.Value(models.ErrGroupKey).(*errgroup.Group) + if !ok { + return 0, fmt.Errorf("failed to get errorgroup from the context") + } + + grp.Go(func() error { + println("Executing the init container command") + cmdErr := utils.ExecuteCommand(ctx, a.logger, cmd, cmdCancel, 25*time.Second) + if cmdErr.Err != nil && cmdErr.Type == utils.Init { + utils.LogError(a.logger, cmdErr.Err, "failed to execute init container command") + } + + println("Init container stopped") + return nil + }) + + // Wait for the container to start or context to cancel + select { + case <-containerStarted: + a.logger.Info("keploy-init container is running") + case <-ctx.Done(): + return 0, fmt.Errorf("context canceled while waiting for container to start") + } + + // Get the PID of the container's first process + inspect, err := a.dockerClient.ContainerInspect(ctx, "keploy-init") + if err != nil { + a.logger.Error("failed to inspect container", zap.Error(err)) + return 0, err + } + + pid := inspect.State.Pid + a.logger.Info("Container PID", zap.Int("pid", pid)) + + // Extract inode from the PID namespace + pidNamespaceInode, err := kdocker.ExtractPidNamespaceInode(pid) + if err != nil { + a.logger.Error("failed to extract PID namespace inode", zap.Error(err)) + return 0, err + } + + a.logger.Info("PID Namespace Inode", zap.String("inode", pidNamespaceInode)) + iNode, err := strconv.ParseUint(pidNamespaceInode, 10, 64) + if err != nil { + a.logger.Error("failed to convert inode to uint64", zap.Error(err)) + return 0, err + } + return iNode, nil +} + +func (a *AgentClient) isAgentRunning(ctx context.Context) bool { + + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://localhost:%d/agent/health", a.conf.Agent.Port), nil) + if err != nil { + utils.LogError(a.logger, err, "failed to send request to the agent server") + } + + resp, err := a.client.Do(req) + if err != nil { + a.logger.Info("Keploy agent is not running in background, starting the agent") + return false + } + a.logger.Info("Setup request sent to the server", zap.String("status", resp.Status)) + return true +} diff --git a/pkg/platform/http/assets/initStop.sh b/pkg/platform/http/assets/initStop.sh new file mode 100755 index 0000000000..87bb6bd30a --- /dev/null +++ b/pkg/platform/http/assets/initStop.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Handle SIGINT and SIGTERM signals, forwarding them to the sleep process +trap 'echo "Init Container received SIGTERM or SIGINT, exiting..."; exit' SIGINT SIGTERM + +# Start sleep infinity to keep the container running in the background +sleep infinity & + +# Wait for the background process and forward any signals +wait $! diff --git a/pkg/platform/yaml/configdb/user/db.go b/pkg/platform/yaml/configdb/user/db.go index 5dc3fae5fa..33b2091221 100644 --- a/pkg/platform/yaml/configdb/user/db.go +++ b/pkg/platform/yaml/configdb/user/db.go @@ -89,16 +89,13 @@ func New(logger *zap.Logger, cfg *config.Config) *Db { func (db *Db) GetInstallationID(_ context.Context) (string, error) { var id string var err error - inDocker := os.Getenv("KEPLOY_INDOCKER") - if inDocker == "true" { - id = os.Getenv("INSTALLATION_ID") - } else { - id, err = machineid.ID() - if err != nil { - db.logger.Debug("failed to get machine id", zap.Error(err)) - return "", nil - } + + id, err = machineid.ID() + if err != nil { + db.logger.Debug("failed to get machine id", zap.Error(err)) + return "", nil } + if id == "" { db.logger.Debug("got empty machine id") return "", nil diff --git a/pkg/service/agent/agent.go b/pkg/service/agent/agent.go new file mode 100644 index 0000000000..7d18ec8179 --- /dev/null +++ b/pkg/service/agent/agent.go @@ -0,0 +1,286 @@ +//go:build linux + +// Package agent contains methods for setting up hooks and proxy along with registering keploy clients. +package agent + +import ( + "context" + "errors" + "fmt" + + "go.keploy.io/server/v2/pkg/core" + "go.keploy.io/server/v2/pkg/core/hooks" + "go.keploy.io/server/v2/pkg/core/hooks/structs" + "go.keploy.io/server/v2/pkg/models" + kdocker "go.keploy.io/server/v2/pkg/platform/docker" + "go.keploy.io/server/v2/utils" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type Agent struct { + logger *zap.Logger + core.Proxy // embedding the Proxy interface to transfer the proxy methods to the core object + core.Hooks // embedding the Hooks interface to transfer the hooks methods to the core object + core.Tester // embedding the Tester interface to transfer the tester methods to the core object + dockerClient kdocker.Client //embedding the docker client to transfer the docker client methods to the core object + proxyStarted bool +} + +func New(logger *zap.Logger, hook core.Hooks, proxy core.Proxy, tester core.Tester, client kdocker.Client) *Agent { + return &Agent{ + logger: logger, + Hooks: hook, + Proxy: proxy, + Tester: tester, + dockerClient: client, + } +} + +// Setup will create a new app and store it in the map, all the setup will be done here +func (a *Agent) Setup(ctx context.Context, _ string, opts models.SetupOptions) error { + + a.logger.Info("Starting the agent in ", zap.String(string(opts.Mode), "mode")) + err := a.Hook(ctx, 0, models.HookOptions{Mode: opts.Mode, IsDocker: opts.IsDocker}) + if err != nil { + a.logger.Error("failed to hook into the app", zap.Error(err)) + } + + select { + case <-ctx.Done(): + fmt.Println("Context cancelled, stopping Agent Setup") + return context.Canceled + } +} + +func (a *Agent) GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) { + return a.Hooks.Record(ctx, id, opts) +} + +func (a *Agent) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) { + m := make(chan *models.Mock, 500) + + err := a.Proxy.Record(ctx, id, m, opts) + if err != nil { + return nil, err + } + + return m, nil +} + +func (a *Agent) MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error { + a.logger.Info("Inside MockOutgoing of agent binary !!") + + err := a.Proxy.Mock(ctx, id, opts) + if err != nil { + return err + } + + return nil +} + +func (a *Agent) Hook(ctx context.Context, id uint64, opts models.HookOptions) error { + hookErr := errors.New("failed to hook into the app") + + select { + case <-ctx.Done(): + fmt.Println("Context cancelled, stopping Hook") + return ctx.Err() + default: + } + + // create a new error group for the hooks + hookErrGrp, _ := errgroup.WithContext(ctx) + hookCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the hookCtx to control the lifecycle of the hooks + hookCtx, hookCtxCancel := context.WithCancel(hookCtx) + hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) + + // create a new error group for the proxy + proxyErrGrp, _ := errgroup.WithContext(ctx) + proxyCtx := context.WithoutCancel(ctx) //so that main context doesn't cancel the proxyCtx to control the lifecycle of the proxy + proxyCtx, proxyCtxCancel := context.WithCancel(proxyCtx) + proxyCtx = context.WithValue(proxyCtx, models.ErrGroupKey, proxyErrGrp) + + hookErrGrp.Go(func() error { + <-ctx.Done() + + proxyCtxCancel() + err := proxyErrGrp.Wait() + if err != nil { + utils.LogError(a.logger, err, "failed to stop the proxy") + } + + hookCtxCancel() + err = hookErrGrp.Wait() + if err != nil { + utils.LogError(a.logger, err, "failed to unload the hooks") + } + return nil + }) + + // load hooks if the mode changes .. + err := a.Hooks.Load(hookCtx, id, core.HookCfg{ + AppID: id, + Pid: 0, + IsDocker: opts.IsDocker, + KeployIPV4: "172.18.0.2", + Mode: opts.Mode, + }) + + if err != nil { + utils.LogError(a.logger, err, "failed to load hooks") + return hookErr + } + + if a.proxyStarted { + a.logger.Info("Proxy already started") + return nil + } + + select { + case <-ctx.Done(): + fmt.Println("Hooks context cancelled, stopping Hook") + return ctx.Err() + default: + } + + // TODO: Hooks can be loaded multiple times but proxy should be started only once + // if there is another containerized app, then we need to pass new (ip:port) of proxy to the eBPF + // as the network namespace is different for each container and so is the keploy/proxy IP to communicate with the app. + err = a.Proxy.StartProxy(proxyCtx, core.ProxyOptions{ + DNSIPv4Addr: "172.18.0.2", + //DnsIPv6Addr: "" + }) + if err != nil { + utils.LogError(a.logger, err, "failed to start proxy") + return hookErr + } + + a.proxyStarted = true + + // For keploy test bench + if opts.EnableTesting { + + // enable testing in the app + // a.EnableTesting = true + // a.Mode = opts.Mode + + // Setting up the test bench + err := a.Tester.Setup(ctx, models.TestingOptions{Mode: opts.Mode}) + if err != nil { + utils.LogError(a.logger, err, "error while setting up the test bench environment") + return errors.New("failed to setup the test bench") + } + } + + return nil +} + +func (a *Agent) SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error { + fmt.Println("Sending Mocks to the Proxy !!") + return a.Proxy.SetMocks(ctx, id, filtered, unFiltered) +} + +func (a *Agent) GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) { + return a.Proxy.GetConsumedMocks(ctx, id) +} + +func (a *Agent) UnHook(_ context.Context, _ uint64) error { + return nil +} + +func (a *Agent) RegisterClient(ctx context.Context, opts models.SetupOptions) error { + fmt.Println("Registering client with keploy client id opts.AppInode!! ", opts.AppInode) + fmt.Println("Registering client with keploy client id opts.ClientInode!! ", opts.ClientInode) + + // send the network info to the kernel + err := a.SendNetworkInfo(ctx, opts) + if err != nil { + a.logger.Error("failed to send network info to the kernel", zap.Error(err)) + return err + } + + clientInfo := structs.ClientInfo{ + KeployClientNsPid: opts.ClientNsPid, + IsDockerApp: 0, + KeployClientInode: opts.ClientInode, + AppInode: opts.AppInode, + } + + switch opts.Mode { + case models.MODE_RECORD: + clientInfo.Mode = uint32(1) + case models.MODE_TEST: + clientInfo.Mode = uint32(2) + default: + clientInfo.Mode = uint32(0) + } + + if opts.IsDocker { + clientInfo.IsDockerApp = 1 + } + + return a.Hooks.SendKeployClientInfo(opts.ClientID, clientInfo) +} + +func (a *Agent) SendNetworkInfo(ctx context.Context, opts models.SetupOptions) error { + if !opts.IsDocker { + proxyIP, err := hooks.IPv4ToUint32("127.0.0.1") + if err != nil { + return err + } + proxyInfo := structs.ProxyInfo{ + IP4: proxyIP, + IP6: [4]uint32{0, 0, 0, 0}, + Port: 16789, + } + err = a.Hooks.SendClientProxyInfo(opts.ClientID, proxyInfo) + if err != nil { + return err + } + return nil + } + + inspect, err := a.dockerClient.ContainerInspect(ctx, "keploy-v2") + if err != nil { + utils.LogError(a.logger, nil, fmt.Sprintf("failed to get inspect keploy container:%v", inspect)) + return err + } + + fmt.Println("OPTS::", opts.DockerNetwork) + keployNetworks := inspect.NetworkSettings.Networks + var keployIPv4 string + for n, settings := range keployNetworks { + if n == "keploy-network" { + keployIPv4 = settings.IPAddress //keploy container IP + break + } + } + fmt.Println("Keploy container IP: ", keployIPv4) + ipv4, err := hooks.IPv4ToUint32(keployIPv4) + if err != nil { + return err + } + + var ipv6 [4]uint32 + if opts.IsDocker { + ipv6, err := hooks.ToIPv4MappedIPv6(keployIPv4) + if err != nil { + return fmt.Errorf("failed to convert ipv4:%v to ipv4 mapped ipv6 in docker env:%v", ipv4, err) + } + a.logger.Debug(fmt.Sprintf("IPv4-mapped IPv6 for %s is: %08x:%08x:%08x:%08x\n", keployIPv4, ipv6[0], ipv6[1], ipv6[2], ipv6[3])) + + } + + proxyInfo := structs.ProxyInfo{ + IP4: ipv4, + IP6: ipv6, + Port: 36789, + } + + err = a.Hooks.SendClientProxyInfo(opts.ClientID, proxyInfo) + if err != nil { + return err + } + return nil +} diff --git a/pkg/service/agent/service.go b/pkg/service/agent/service.go new file mode 100644 index 0000000000..a707e27608 --- /dev/null +++ b/pkg/service/agent/service.go @@ -0,0 +1,34 @@ +package agent + +import ( + "context" + + "go.keploy.io/server/v2/pkg/models" +) + +type Service interface { + Setup(ctx context.Context, cmd string, opts models.SetupOptions) error + GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) + GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) + MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error + SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error + GetConsumedMocks(ctx context.Context, id uint64) ([]string, error) + RegisterClient(ctx context.Context, opts models.SetupOptions) error +} + +type Options struct { + // Platform Platform + Network string + Container string + SelfTesting bool + Mode models.Mode +} + +// type Platform string + +// var ( +// linux Platform = "linux" +// windows Platform = "windows" +// mac Platform = "mac" +// docker Platform = "docker" +// ) diff --git a/pkg/service/agent/utils.go b/pkg/service/agent/utils.go new file mode 100644 index 0000000000..ca44fb8631 --- /dev/null +++ b/pkg/service/agent/utils.go @@ -0,0 +1,3 @@ +// Package agent contains utilities for agent. + +package agent diff --git a/pkg/service/orchestrator/orchestrator.go b/pkg/service/orchestrator/orchestrator.go index b07150fbde..16b8533d86 100644 --- a/pkg/service/orchestrator/orchestrator.go +++ b/pkg/service/orchestrator/orchestrator.go @@ -1,4 +1,3 @@ -//go:build linux // Package orchestrator acts as a main brain for both the record and replay services package orchestrator diff --git a/pkg/service/orchestrator/rerecord.go b/pkg/service/orchestrator/rerecord.go index 2f56eb9ccf..33a4e56cb0 100644 --- a/pkg/service/orchestrator/rerecord.go +++ b/pkg/service/orchestrator/rerecord.go @@ -1,4 +1,3 @@ -//go:build linux package orchestrator @@ -207,7 +206,7 @@ func (o *Orchestrator) replayTests(ctx context.Context, testSet string) (bool, e if utils.IsDockerCmd(cmdType) { host = o.config.ContainerName - userIP, err = o.record.GetContainerIP(ctx, o.config.AppID) + userIP, err = o.replay.GetContainerIP(ctx, o.config.AppID) if err != nil { utils.LogError(o.logger, err, "failed to get the app ip") return false, err diff --git a/pkg/service/record/record.go b/pkg/service/record/record.go index 78712fa56d..f0c70e9198 100755 --- a/pkg/service/record/record.go +++ b/pkg/service/record/record.go @@ -1,5 +1,3 @@ -//go:build linux - // Package record provides functionality for recording and managing test cases and mocks. package record @@ -49,10 +47,11 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { runAppCtx := context.WithoutCancel(ctx) runAppCtx, runAppCtxCancel := context.WithCancel(runAppCtx) - hookErrGrp, _ := errgroup.WithContext(ctx) - hookCtx := context.WithoutCancel(ctx) - hookCtx, hookCtxCancel := context.WithCancel(hookCtx) - hookCtx = context.WithValue(hookCtx, models.ErrGroupKey, hookErrGrp) + setupErrGrp, _ := errgroup.WithContext(ctx) + setupCtx := context.WithoutCancel(ctx) + setupCtx, setupCtxCancel := context.WithCancel(setupCtx) + setupCtx = context.WithValue(ctx, models.ErrGroupKey, setupErrGrp) + // reRecordCtx, reRecordCancel := context.WithCancel(ctx) // defer reRecordCancel() // Cancel the context when the function returns @@ -72,6 +71,7 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { defer func() { select { case <-ctx.Done(): + fmt.Println("Context cancelled start ") default: if !reRecord { err := utils.Stop(r.logger, stopReason) @@ -85,10 +85,11 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { if err != nil { utils.LogError(r.logger, err, "failed to stop application") } - hookCtxCancel() - err = hookErrGrp.Wait() + + setupCtxCancel() + err = setupErrGrp.Wait() if err != nil { - utils.LogError(r.logger, err, "failed to stop hooks") + utils.LogError(r.logger, err, "failed to stop setup execution, that covers init container") } err = errGrp.Wait() if err != nil { @@ -98,8 +99,6 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { }() defer close(appErrChan) - defer close(insertTestErrChan) - defer close(insertMockErrChan) newTestSetID, err := r.GetNextTestSetID(ctx) if err != nil { @@ -111,14 +110,15 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { //checking for context cancellation as we don't want to start the instrumentation if the context is cancelled select { case <-ctx.Done(): + fmt.Println("Context cancelled 0") return nil default: } - // Instrument will setup the environment and start the hooks and proxy - appID, err = r.Instrument(hookCtx) + // setting up the environment for recording + appID, err = r.instrumentation.Setup(setupCtx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay, Mode: models.MODE_RECORD, CommandType: r.config.CommandType}) if err != nil { - stopReason = "failed to instrument the application" + stopReason = "failed setting up the environment" utils.LogError(r.logger, err, stopReason) return fmt.Errorf(stopReason) } @@ -193,6 +193,7 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { return errors.New("failed to stop recording") } case <-ctx.Done(): + fmt.Println("Context cancelled 1") return nil } return nil @@ -227,60 +228,62 @@ func (r *Recorder) Start(ctx context.Context, reRecord bool) error { case err = <-insertMockErrChan: stopReason = "error while inserting mock into db, hence stopping keploy" case <-ctx.Done(): + fmt.Println("Context cancelled 2") return nil } utils.LogError(r.logger, err, stopReason) return fmt.Errorf(stopReason) } -func (r *Recorder) Instrument(ctx context.Context) (uint64, error) { - var stopReason string - - // setting up the environment for recording - appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay}) - if err != nil { - stopReason = "failed setting up the environment" - utils.LogError(r.logger, err, stopReason) - return 0, fmt.Errorf(stopReason) - } - r.config.AppID = appID - - // checking for context cancellation as we don't want to start the hooks and proxy if the context is cancelled - select { - case <-ctx.Done(): - return appID, nil - default: - // Starting the hooks and proxy - err = r.instrumentation.Hook(ctx, appID, models.HookOptions{Mode: models.MODE_RECORD, EnableTesting: r.config.EnableTesting, Rules: r.config.BypassRules}) - if err != nil { - stopReason = "failed to start the hooks and proxy" - utils.LogError(r.logger, err, stopReason) - if ctx.Err() == context.Canceled { - return appID, err - } - return appID, fmt.Errorf(stopReason) - } - } - return appID, nil -} - func (r *Recorder) GetTestAndMockChans(ctx context.Context, appID uint64) (FrameChan, error) { incomingOpts := models.IncomingOptions{ Filters: r.config.Record.Filters, } - incomingChan, err := r.instrumentation.GetIncoming(ctx, appID, incomingOpts) - if err != nil { - return FrameChan{}, fmt.Errorf("failed to get incoming test cases: %w", err) - } outgoingOpts := models.OutgoingOptions{ Rules: r.config.BypassRules, MongoPassword: r.config.Test.MongoPassword, FallBackOnMiss: r.config.Test.FallBackOnMiss, } - outgoingChan, err := r.instrumentation.GetOutgoing(ctx, appID, outgoingOpts) - if err != nil { - return FrameChan{}, fmt.Errorf("failed to get outgoing mocks: %w", err) + + // Create channels to receive incoming and outgoing data + incomingChan := make(chan *models.TestCase) + outgoingChan := make(chan *models.Mock) + errChan := make(chan error, 2) + + go func() { + defer close(incomingChan) + ch, err := r.instrumentation.GetIncoming(ctx, appID, incomingOpts) + if err != nil { + errChan <- fmt.Errorf("failed to get incoming test cases: %w", err) + return + } + + for testCase := range ch { + incomingChan <- testCase + } + }() + + go func() { + defer close(outgoingChan) + ch, err := r.instrumentation.GetOutgoing(ctx, appID, outgoingOpts) + if err != nil { + errChan <- fmt.Errorf("failed to get outgoing mocks: %w", err) + return + } + + for mock := range ch { + outgoingChan <- mock + } + }() + + // Check for errors after starting the goroutines + select { + case err := <-errChan: + // If there's an error, return it immediately + return FrameChan{}, err + default: + // No errors, proceed } return FrameChan{ diff --git a/pkg/service/record/service.go b/pkg/service/record/service.go index 700ca53bf2..263f66e44e 100755 --- a/pkg/service/record/service.go +++ b/pkg/service/record/service.go @@ -10,7 +10,6 @@ type Instrumentation interface { //Setup prepares the environment for the recording Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) //Hook will load hooks and start the proxy server. - Hook(ctx context.Context, id uint64, opts models.HookOptions) error GetIncoming(ctx context.Context, id uint64, opts models.IncomingOptions) (<-chan *models.TestCase, error) GetOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) (<-chan *models.Mock, error) // Run is blocking call and will execute until error @@ -20,7 +19,6 @@ type Instrumentation interface { type Service interface { Start(ctx context.Context, reRecord bool) error - GetContainerIP(ctx context.Context, id uint64) (string, error) } type TestDB interface { diff --git a/pkg/service/record/utils.go b/pkg/service/record/utils.go index aa96e8bc5c..6162eca075 100644 --- a/pkg/service/record/utils.go +++ b/pkg/service/record/utils.go @@ -1,3 +1 @@ -//go:build linux - package record diff --git a/pkg/service/replay/replay.go b/pkg/service/replay/replay.go index 37a6308bed..9833fcc208 100644 --- a/pkg/service/replay/replay.go +++ b/pkg/service/replay/replay.go @@ -81,7 +81,11 @@ func (r *Replayer) Start(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) ctx = context.WithValue(ctx, models.ErrGroupKey, g) - var hookCancel context.CancelFunc + setupErrGrp, _ := errgroup.WithContext(ctx) + setupCtx := context.WithoutCancel(ctx) + setupCtx = context.WithValue(ctx, models.ErrGroupKey, setupErrGrp) + setupCtx, setupCtxCancel := context.WithCancel(setupCtx) + var stopReason = "replay completed successfully" // defering the stop function to stop keploy in case of any error in record or in case of context cancellation @@ -92,10 +96,17 @@ func (r *Replayer) Start(ctx context.Context) error { default: r.logger.Info("stopping Keploy", zap.String("reason", stopReason)) } - if hookCancel != nil { - hookCancel() + + fmt.Printf("SetupCtx?: %v\n", setupCtx.Err()) + setupCtxCancel() + fmt.Printf("SetupCtx?: %v\n", setupCtx.Err()) + println("setupCtxCancel is cancelled") + err := setupErrGrp.Wait() + if err != nil { + utils.LogError(r.logger, err, "failed to stop setup execution, that covers init container") } - err := g.Wait() + + err = g.Wait() if err != nil { utils.LogError(r.logger, err, "failed to stop replaying") } @@ -172,7 +183,7 @@ func (r *Replayer) Start(ctx context.Context) error { } // Instrument will load the hooks and start the proxy - inst, err := r.Instrument(ctx) + inst, err := r.Instrument(setupCtx) if err != nil { stopReason = fmt.Sprintf("failed to instrument: %v", err) utils.LogError(r.logger, err, stopReason) @@ -182,8 +193,6 @@ func (r *Replayer) Start(ctx context.Context) error { return fmt.Errorf(stopReason) } - hookCancel = inst.HookCancel - var testSetResult bool testRunResult := true abortTestRun := false @@ -331,33 +340,19 @@ func (r *Replayer) Instrument(ctx context.Context) (*InstrumentState, error) { r.logger.Info("Keploy will not mock the outgoing calls when base path is provided", zap.Any("base path", r.config.Test.BasePath)) return &InstrumentState{}, nil } - appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, DockerDelay: r.config.BuildDelay}) + // Instrument will setup the environment and start the hooks and proxy + appID, err := r.instrumentation.Setup(ctx, r.config.Command, models.SetupOptions{Container: r.config.ContainerName, DockerNetwork: r.config.NetworkName, CommandType: r.config.CommandType, DockerDelay: r.config.BuildDelay, Mode: models.MODE_TEST}) if err != nil { - if errors.Is(err, context.Canceled) { - return &InstrumentState{}, err - } - return &InstrumentState{}, fmt.Errorf("failed to setup instrumentation: %w", err) + stopReason := "failed setting up the environment" + utils.LogError(r.logger, err, stopReason) + return &InstrumentState{}, fmt.Errorf(stopReason) } + r.config.AppID = appID var cancel context.CancelFunc - // starting the hooks and proxy - select { - case <-ctx.Done(): - return &InstrumentState{}, context.Canceled - default: - hookCtx := context.WithoutCancel(ctx) - hookCtx, cancel = context.WithCancel(hookCtx) - err = r.instrumentation.Hook(hookCtx, appID, models.HookOptions{Mode: models.MODE_TEST, EnableTesting: r.config.EnableTesting, Rules: r.config.BypassRules}) - if err != nil { - cancel() - if errors.Is(err, context.Canceled) { - return &InstrumentState{}, err - } - return &InstrumentState{}, fmt.Errorf("failed to start the hooks and proxy: %w", err) - } - } - return &InstrumentState{AppID: appID, HookCancel: cancel}, nil + + return &InstrumentState{AppID: 0, HookCancel: cancel}, nil } func (r *Replayer) GetNextTestRunID(ctx context.Context) (string, error) { @@ -833,6 +828,8 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s if r.config.Test.UpdateTemplate || r.config.Test.BasePath != "" { removeDoubleQuotes(utils.TemplatizedValues) // Write the templatized values to the yaml. + fmt.Println("POSTScript", conf.PostScript) + fmt.Println("PREScript", conf.PreScript) if len(utils.TemplatizedValues) > 0 { err = r.testSetConf.Write(ctx, testSetID, &models.TestSet{ PreScript: conf.PreScript, @@ -876,6 +873,7 @@ func (r *Replayer) SetupOrUpdateMocks(ctx context.Context, appID uint64, testSet } if action == Start { + // api call here - err = r.instrumentation.MockOutgoing(ctx, appID, models.OutgoingOptions{ Rules: r.config.BypassRules, MongoPassword: r.config.Test.MongoPassword, @@ -889,6 +887,7 @@ func (r *Replayer) SetupOrUpdateMocks(ctx context.Context, appID uint64, testSet } } + // this will be sent to the proxy err = r.instrumentation.SetMocks(ctx, appID, filteredMocks, unfilteredMocks) if err != nil { utils.LogError(r.logger, err, "failed to set mocks") @@ -1142,3 +1141,7 @@ func (r *Replayer) DeleteTests(ctx context.Context, testSetID string, testCaseID func SetTestHooks(testHooks TestHooks) { HookImpl = testHooks } + +func (r *Replayer) GetContainerIP(ctx context.Context, id uint64) (string, error) { + return r.instrumentation.GetContainerIP(ctx, id) +} diff --git a/pkg/service/replay/service.go b/pkg/service/replay/service.go index 5ffa0414b1..02e5dc5dc3 100644 --- a/pkg/service/replay/service.go +++ b/pkg/service/replay/service.go @@ -12,7 +12,7 @@ type Instrumentation interface { //Setup prepares the environment for the recording Setup(ctx context.Context, cmd string, opts models.SetupOptions) (uint64, error) //Hook will load hooks and start the proxy server. - Hook(ctx context.Context, id uint64, opts models.HookOptions) error + // Hook(ctx context.Context, id uint64, opts models.HookOptions) error MockOutgoing(ctx context.Context, id uint64, opts models.OutgoingOptions) error // SetMocks Allows for setting mocks between test runs for better filtering and matching SetMocks(ctx context.Context, id uint64, filtered []*models.Mock, unFiltered []*models.Mock) error @@ -26,9 +26,10 @@ type Instrumentation interface { type Service interface { Start(ctx context.Context) error - Instrument(ctx context.Context) (*InstrumentState, error) + // Instrument(ctx context.Context) (*InstrumentState, error) GetNextTestRunID(ctx context.Context) (string, error) GetAllTestSetIDs(ctx context.Context) ([]string, error) + GetContainerIP(ctx context.Context, id uint64) (string, error) RunTestSet(ctx context.Context, testSetID string, testRunID string, appID uint64, serveTest bool) (models.TestSetStatus, error) GetTestSetStatus(ctx context.Context, testRunID string, testSetID string) (models.TestSetStatus, error) GetTestCases(ctx context.Context, testID string) ([]*models.TestCase, error) diff --git a/pkg/service/utgen/assets/test_generation.toml b/pkg/service/utgen/assets/test_generation.toml index 63b7c5b884..10a7cfa2da 100644 --- a/pkg/service/utgen/assets/test_generation.toml +++ b/pkg/service/utgen/assets/test_generation.toml @@ -30,6 +30,13 @@ Here is the file that contains the existing tests, called `{{ .test_file_name }} {{ .test_file | trim }} ========= +## Installed Packages +The following packages are already installed in the environment. Use these when writing tests to avoid redundant installations: + +========= +{{ .installed_packages | trim }} +========= + {%- if additional_includes_section | trim %} {{ .additional_includes_section | trim }} @@ -65,12 +72,13 @@ class SingleTest(BaseModel): {%- endif %} test_code: str = Field(description="A single test function, that tests the behavior described in 'test_behavior'. The test should be a written like its a part of the existing test suite, if there is one, and it can use existing helper functions, setup, or teardown code.") new_imports_code: str = Field(description="Code for new imports that are required for the new test function, and are not already present in the test file. Give an empty string if no new imports are required.") + library_installation_code: str = Field(description="If new libraries are needed, specify the installation commands. Provide an empty string if no new libraries are required.") test_tags: str = Field(description="A single label that best describes the test, out of: ['happy path', 'edge case','other']") class NewTests(BaseModel): language: str = Field(description="The programming language of the source code") existing_test_function_signature: str = Field(description="A single line repeating a signature header of one of the existing test functions") - new_tests: List[SingleTest] = Field(min_items=1, max_items={{ .max_tests }}, description="A list of new test functions to append to the existing test suite, aiming to increase the code coverage. Each test should run as-is, without requiring any additional inputs or setup code. Don't introduce new dependencies") + new_tests: List[SingleTest] = Field(min_items=1, max_items={{ .max_tests }}, description="A list of new test functions to append to the existing test suite, aiming to increase the code coverage. Each test should run as-is, without requiring any additional inputs or setup code.") ===== Example output: @@ -95,7 +103,40 @@ new_tests: ... {%- endif %} new_imports_code: | - "" + {%- if language == "python" %} + - "import pytest" + - "from my_module import my_function" + {%- elif language == "java" %} + - "import org.junit.jupiter.api.Test;" + - "import my.package.MyFunction;" + {%- elif language == "golang" %} + - "import \"testing\"" + - "import \"my_module\"" + {%- elif language == "javascript" %} + - "const assert = require('assert');" + - "const myFunction = require('my_module').myFunction;" + {%- elif language == "typescript" %} + - "import { assert } from 'assert';" + - "import { myFunction } from 'my_module';" + {%- endif %} + library_installation_code: | + {%- if language == "python" %} + - pip install pytest + {%- elif language == "java" %} + # Add the following to your Maven pom.xml: + # + # org.junit.jupiter + # junit-jupiter-api + # 5.7.0 + # test + # + {%- elif language == "golang" %} + - go get github.com/my_module + {%- elif language == "javascript" %} + - npm install assert + {%- elif language == "typescript" %} + - npm install assert + {%- endif %} test_tags: happy path ... ``` diff --git a/pkg/service/utgen/gen.go b/pkg/service/utgen/gen.go index 7138ee3076..c485d400f4 100644 --- a/pkg/service/utgen/gen.go +++ b/pkg/service/utgen/gen.go @@ -2,10 +2,15 @@ package utgen import ( + "bufio" "context" "fmt" + "io/fs" "math" "os" + "os/exec" + "regexp" + "sort" "strings" "github.com/google/uuid" @@ -53,27 +58,387 @@ type UnitTestGenerator struct { noCoverageTest int } -func NewUnitTestGenerator(srcPath, testPath, reportPath, cmd, dir, coverageFormat string, desiredCoverage float64, maxIterations int, model string, apiBaseURL string, apiVersion, apiServerURL, additionalPrompt string, _ *config.Config, tel Telemetry, auth service.Auth, logger *zap.Logger) (*UnitTestGenerator, error) { +func NewUnitTestGenerator( + cfg *config.Config, + tel Telemetry, + auth service.Auth, + logger *zap.Logger, +) (*UnitTestGenerator, error) { + genConfig := cfg.Gen + generator := &UnitTestGenerator{ - srcPath: srcPath, - testPath: testPath, - cmd: cmd, - dir: dir, - maxIterations: maxIterations, + srcPath: genConfig.SourceFilePath, + testPath: genConfig.TestFilePath, + cmd: genConfig.TestCommand, + dir: genConfig.TestDir, + maxIterations: genConfig.MaxIterations, logger: logger, tel: tel, - ai: NewAIClient(model, apiBaseURL, apiVersion, "", apiServerURL, auth, uuid.NewString(), logger), + ai: NewAIClient(genConfig.Model, genConfig.APIBaseURL, genConfig.APIVersion, "", cfg.APIServerURL, auth, uuid.NewString(), logger), cov: &Coverage{ - Path: reportPath, - Format: coverageFormat, - Desired: desiredCoverage, + Path: genConfig.CoverageReportPath, + Format: genConfig.CoverageFormat, + Desired: genConfig.DesiredCoverage, }, - additionalPrompt: additionalPrompt, + additionalPrompt: genConfig.AdditionalPrompt, cur: &Cursor{}, } return generator, nil } +func updateJavaScriptImports(importedContent string, newImports []string) (string, int, error) { + importRegex := regexp.MustCompile(`(?m)^(import\s+.*?from\s+['"].*?['"];?|const\s+.*?=\s+require\(['"].*?['"]\);?)`) + existingImportsSet := make(map[string]bool) + + existingImports := importRegex.FindAllString(importedContent, -1) + for _, imp := range existingImports { + if imp != "\"\"" && len(imp) > 0 { + existingImportsSet[imp] = true + } + } + + for _, imp := range newImports { + imp = strings.TrimSpace(imp) + if imp != "\"\"" && len(imp) > 0 { + existingImportsSet[imp] = true + } + } + + allImports := make([]string, 0, len(existingImportsSet)) + for imp := range existingImportsSet { + allImports = append(allImports, imp) + } + + importSection := strings.Join(allImports, "\n") + + updatedContent := importRegex.ReplaceAllString(importedContent, "") + updatedContent = strings.Trim(updatedContent, "\n") + lines := strings.Split(updatedContent, "\n") + cleanedLines := []string{} + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine != "" { + cleanedLines = append(cleanedLines, line) + } + } + updatedContent = strings.Join(cleanedLines, "\n") + updatedContent = importSection + "\n" + updatedContent + + importLength := len(strings.Split(updatedContent, "\n")) - len(strings.Split(importedContent, "\n")) + if importLength < 0 { + importLength = 0 + } + return updatedContent, importLength, nil +} + +func updateImports(filePath string, language string, imports string) (int, error) { + newImports := strings.Split(imports, "\n") + for i, imp := range newImports { + newImports[i] = strings.TrimSpace(imp) + } + contentBytes, err := os.ReadFile(filePath) + if err != nil { + return 0, err + } + content := string(contentBytes) + + var updatedContent string + var importLength int + switch strings.ToLower(language) { + case "go": + updatedContent, importLength, err = updateGoImports(content, newImports) + case "java": + updatedContent, err = updateJavaImports(content, newImports) + case "python": + updatedContent, err = updatePythonImports(content, newImports) + case "typescript": + updatedContent, importLength, err = updateTypeScriptImports(content, newImports) + case "javascript": + updatedContent, importLength, err = updateJavaScriptImports(content, newImports) + default: + return 0, fmt.Errorf("unsupported language: %s", language) + } + if err != nil { + return 0, err + } + err = os.WriteFile(filePath, []byte(updatedContent), fs.ModePerm) + + if err != nil { + return 0, err + } + + return importLength, nil +} + +func updateGoImports(codeBlock string, newImports []string) (string, int, error) { + importRegex := regexp.MustCompile(`(?ms)import\s*(\([\s\S]*?\)|"[^"]+")`) + existingImportsSet := make(map[string]bool) + + matches := importRegex.FindStringSubmatch(codeBlock) + if matches != nil { + importBlock := matches[0] + importLines := strings.Split(importBlock, "\n") + existingImports := extractGoImports(importLines) + for _, imp := range existingImports { + existingImportsSet[imp] = true + } + newImports = extractGoImports(newImports) + for _, imp := range newImports { + imp = strings.TrimSpace(imp) + if imp != "\"\"" && len(imp) > 0 { + existingImportsSet[imp] = true + } + } + allImports := make([]string, 0, len(existingImportsSet)) + for imp := range existingImportsSet { + allImports = append(allImports, imp) + } + importBlockNew := createGoImportBlock(allImports) + updatedContent := importRegex.ReplaceAllString(codeBlock, importBlockNew) + return updatedContent, len(strings.Split(importBlockNew, "\n")) - len(importLines), nil + } + packageRegex := regexp.MustCompile(`package\s+\w+`) + + pkgMatch := packageRegex.FindStringIndex(codeBlock) + if pkgMatch == nil { + return "", 0, fmt.Errorf("could not find package declaration") + } + newImports = extractGoImports(newImports) + importBlock := createGoImportBlock(newImports) + + insertPos := pkgMatch[1] + updatedContent := codeBlock[:insertPos] + "\n\n" + importBlock + "\n" + codeBlock[insertPos:] + return updatedContent, len(strings.Split(importBlock, "\n")) + 1, nil + +} + +func extractGoImports(importLines []string) []string { + imports := []string{} + for _, line := range importLines { + line = strings.TrimSpace(line) + if (line == "import (" || line == ")" || line == "") || len(line) == 0 { + continue + } + line = strings.TrimPrefix(line, "import ") + line = strings.Trim(line, `"`) + imports = append(imports, line) + } + return imports +} + +func createGoImportBlock(imports []string) string { + importBlock := "import (\n" + for _, imp := range imports { + imp = strings.TrimSpace(imp) + imp = strings.Trim(imp, `"`) + importBlock += fmt.Sprintf(` "%s"`+"\n", imp) + } + importBlock += ")" + return importBlock +} + +func updateJavaImports(content string, newImports []string) (string, error) { + importRegex := regexp.MustCompile(`(?m)^import\s+.*?;`) + existingImportsSet := make(map[string]bool) + + existingImports := importRegex.FindAllString(content, -1) + for _, imp := range existingImports { + existingImportsSet[imp] = true + } + + for _, imp := range newImports { + imp = strings.TrimSpace(imp) + if imp != "\"\"" && len(imp) > 0 { + importStatement := fmt.Sprintf("import %s;", imp) + existingImportsSet[importStatement] = true + } + } + + allImports := make([]string, 0, len(existingImportsSet)) + for imp := range existingImportsSet { + allImports = append(allImports, imp) + } + importSection := strings.Join(allImports, "\n") + + updatedContent := importRegex.ReplaceAllString(content, "") + packageRegex := regexp.MustCompile(`(?m)^package\s+.*?;`) + pkgMatch := packageRegex.FindStringIndex(updatedContent) + insertPos := 0 + if pkgMatch != nil { + insertPos = pkgMatch[1] + } + + updatedContent = updatedContent[:insertPos] + "\n\n" + importSection + "\n" + updatedContent[insertPos:] + return updatedContent, nil +} + +func updatePythonImports(content string, newImports []string) (string, error) { + scanner := bufio.NewScanner(strings.NewReader(content)) + existingImportsMap := make(map[string]map[string]bool) + codeLines := []string{} + importLines := []string{} + + ignoredPrefixes := "# checking coverage for file - do not remove" + + for scanner.Scan() { + line := scanner.Text() + trimmedLine := strings.TrimSpace(line) + + if trimmedLine == "" { + continue + } + shouldIgnore := (strings.HasPrefix(trimmedLine, "import ") || strings.HasPrefix(trimmedLine, "from ")) && strings.Contains(trimmedLine, ignoredPrefixes) + if shouldIgnore { + parts := strings.Split(trimmedLine, "#") + coreImport := strings.TrimSpace(parts[0]) + + if strings.HasPrefix(coreImport, "from ") { + fields := strings.Fields(coreImport) + moduleName := fields[1] + importPart := coreImport[strings.Index(coreImport, "import")+len("import "):] + importedItems := strings.Split(importPart, ",") + + if _, exists := existingImportsMap[moduleName]; !exists { + existingImportsMap[moduleName] = make(map[string]bool) + } + for _, item := range importedItems { + cleanedItem := strings.TrimSpace(item) + if cleanedItem != "" { + existingImportsMap[moduleName][cleanedItem] = true + } + } + } + codeLines = append(codeLines, line) + continue + } + + if strings.HasPrefix(trimmedLine, "import ") || strings.HasPrefix(trimmedLine, "from ") { + codeLines = append(codeLines, line) + } else { + codeLines = append(codeLines, line) + } + } + + for _, imp := range newImports { + imp = strings.TrimSpace(imp) + if imp == "\"\"" || len(imp) == 0 { + continue + } + if strings.HasPrefix(imp, "from ") { + fields := strings.Fields(imp) + moduleName := fields[1] + newItems := strings.Split(fields[3], ",") + if _, exists := existingImportsMap[moduleName]; !exists { + existingImportsMap[moduleName] = make(map[string]bool) + } + for _, item := range newItems { + existingImportsMap[moduleName][strings.TrimSpace(item)] = true + } + } else if strings.HasPrefix(imp, "import ") { + fields := strings.Fields(imp) + moduleName := fields[1] + if _, exists := existingImportsMap[moduleName]; !exists { + existingImportsMap[moduleName] = make(map[string]bool) + } + } + } + for i, line := range codeLines { + trimmedLine := strings.TrimSpace(line) + + if strings.HasPrefix(trimmedLine, "from ") { + fields := strings.Fields(trimmedLine) + moduleName := fields[1] + + if itemsMap, exists := existingImportsMap[moduleName]; exists && len(itemsMap) > 0 { + items := mapKeysToSortedSlice(itemsMap) + importLine := fmt.Sprintf("from %s import %s", moduleName, strings.Join(items, ", ")) + + if strings.Contains(trimmedLine, ignoredPrefixes) { + importLine += " " + ignoredPrefixes + } + codeLines[i] = importLine + delete(existingImportsMap, moduleName) + } + } + } + + for module, itemsMap := range existingImportsMap { + if len(itemsMap) > 0 { + items := mapKeysToSortedSlice(itemsMap) + importLine := fmt.Sprintf("from %s import %s", module, strings.Join(items, ", ")) + importLine += " " + ignoredPrefixes + importLines = append(importLines, importLine) + } + } + nonEmptyCodeLines := []string{} + for _, line := range codeLines { + if strings.TrimSpace(line) != "" { + nonEmptyCodeLines = append(nonEmptyCodeLines, line) + } + } + + nonEmptyImportLines := []string{} + for _, line := range importLines { + if strings.TrimSpace(line) != "" { + nonEmptyImportLines = append(nonEmptyImportLines, line) + } + } + + updatedContent := strings.Join(nonEmptyImportLines, "\n") + "\n" + strings.Join(nonEmptyCodeLines, "\n") + return updatedContent, nil +} + +// Helper function to convert map keys to a sorted slice +func mapKeysToSortedSlice(itemsMap map[string]bool) []string { + items := []string{} + for item := range itemsMap { + items = append(items, item) + } + sort.Strings(items) + return items +} + +func updateTypeScriptImports(importedContent string, newImports []string) (string, int, error) { + importRegex := regexp.MustCompile(`(?m)^import\s+.*?;`) + existingImportsSet := make(map[string]bool) + + existingImports := importRegex.FindAllString(importedContent, -1) + for _, imp := range existingImports { + existingImportsSet[imp] = true + } + + for _, imp := range newImports { + imp = strings.TrimSpace(imp) + if imp != "\"\"" && len(imp) > 0 { + existingImportsSet[imp] = true + } + } + + allImports := make([]string, 0, len(existingImportsSet)) + for imp := range existingImportsSet { + allImports = append(allImports, imp) + } + importSection := strings.Join(allImports, "\n") + + updatedContent := importRegex.ReplaceAllString(importedContent, "") + updatedContent = strings.Trim(updatedContent, "\n") + lines := strings.Split(updatedContent, "\n") + cleanedLines := []string{} + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine != "" { + cleanedLines = append(cleanedLines, line) + } + } + updatedContent = strings.Join(cleanedLines, "\n") + updatedContent = importSection + "\n" + updatedContent + importLength := len(strings.Split(updatedContent, "\n")) - len(strings.Split(importedContent, "\n")) + if importLength < 0 { + importLength = 0 + } + return updatedContent, importLength, nil +} + func (g *UnitTestGenerator) Start(ctx context.Context) error { g.tel.GenerateUT() @@ -189,6 +554,10 @@ func (g *UnitTestGenerator) Start(ctx context.Context) error { } } + g.promptBuilder.InstalledPackages, err = libraryInstalled(g.logger, g.lang) + if err != nil { + utils.LogError(g.logger, err, "Error getting installed packages") + } g.prompt, err = g.promptBuilder.BuildPrompt("test_generation", failedTestRunsValue) if err != nil { utils.LogError(g.logger, err, "Error building prompt") @@ -205,12 +574,16 @@ func (g *UnitTestGenerator) Start(ctx context.Context) error { g.totalTestCase += len(testsDetails.NewTests) totalTest = len(testsDetails.NewTests) for _, generatedTest := range testsDetails.NewTests { + installedPackages, err := libraryInstalled(g.logger, g.lang) + if err != nil { + g.logger.Warn("Error getting installed packages", zap.Error(err)) + } select { case <-ctx.Done(): return fmt.Errorf("process cancelled by user") default: } - err := g.ValidateTest(generatedTest, &passedTests, &noCoverageTest, &failedBuild) + err = g.ValidateTest(generatedTest, &passedTests, &noCoverageTest, &failedBuild, installedPackages) if err != nil { utils.LogError(g.logger, err, "Error validating test") return err @@ -491,7 +864,7 @@ func (g *UnitTestGenerator) getLine(ctx context.Context) (int, error) { return line, nil } -func (g *UnitTestGenerator) ValidateTest(generatedTest models.UT, passedTests, noCoverageTest, failedBuild *int) error { +func (g *UnitTestGenerator) ValidateTest(generatedTest models.UT, passedTests, noCoverageTest, failedBuild *int, installedPackages []string) error { testCode := strings.TrimSpace(generatedTest.TestCode) InsertAfter := g.cur.Line Indent := g.cur.Indentation @@ -515,6 +888,9 @@ func (g *UnitTestGenerator) ValidateTest(generatedTest models.UT, passedTests, n } originalContentLines := strings.Split(originalContent, "\n") testCodeLines := strings.Split(testCodeIndented, "\n") + if InsertAfter > len(originalContentLines) { + InsertAfter = len(originalContentLines) + } processedTestLines := append(originalContentLines[:InsertAfter], testCodeLines...) processedTestLines = append(processedTestLines, originalContentLines[InsertAfter:]...) processedTest := strings.Join(processedTestLines, "\n") @@ -522,11 +898,19 @@ func (g *UnitTestGenerator) ValidateTest(generatedTest models.UT, passedTests, n return fmt.Errorf("failed to write test file: %w", err) } + newInstalledPackages, err := installLibraries(generatedTest.LibraryInstallationCode, installedPackages, g.logger) + if err != nil { + g.logger.Debug("Error installing libraries", zap.Error(err)) + } + // Run the test using the Runner class g.logger.Info(fmt.Sprintf("Running test 5 times for proper validation with the following command: '%s'", g.cmd)) var testCommandStartTime int64 - + importLen, err := updateImports(g.testPath, g.lang, generatedTest.NewImportsCode) + if err != nil { + g.logger.Warn("Error updating imports", zap.Error(err)) + } for i := 0; i < 5; i++ { g.logger.Info(fmt.Sprintf("Iteration no: %d", i+1)) @@ -543,6 +927,11 @@ func (g *UnitTestGenerator) ValidateTest(generatedTest models.UT, passedTests, n if err := os.WriteFile(g.testPath, []byte(originalContent), 0644); err != nil { return fmt.Errorf("failed to write test file: %w", err) } + err = uninstallLibraries(g.lang, newInstalledPackages, g.logger) + + if err != nil { + g.logger.Warn("Error uninstalling libraries", zap.Error(err)) + } g.logger.Info("Skipping a generated test that failed") g.failedTests = append(g.failedTests, &models.FailedUT{ TestCode: generatedTest.TestCode, @@ -573,13 +962,193 @@ func (g *UnitTestGenerator) ValidateTest(generatedTest models.UT, passedTests, n if err := os.WriteFile(g.testPath, []byte(originalContent), 0644); err != nil { return fmt.Errorf("failed to write test file: %w", err) } + + err = uninstallLibraries(g.lang, newInstalledPackages, g.logger) + + if err != nil { + g.logger.Warn("Error uninstalling libraries", zap.Error(err)) + } + g.logger.Info("Skipping a generated test that failed to increase coverage") return nil } g.testCasePassed++ *passedTests++ g.cov.Current = covResult.Coverage + g.cur.Line = g.cur.Line + len(testCodeLines) + importLen g.cur.Line = g.cur.Line + len(testCodeLines) g.logger.Info("Generated test passed and increased coverage") return nil } + +func libraryInstalled(logger *zap.Logger, language string) ([]string, error) { + switch strings.ToLower(language) { + case "go": + out, err := exec.Command("go", "list", "-m", "all").Output() + if err != nil { + return nil, fmt.Errorf("failed to get Go dependencies: %w", err) + } + return extractDependencies(out), nil + + case "java": + out, err := exec.Command("mvn", "dependency:list", "-DincludeScope=compile", "-Dstyle.color=never", "-B").Output() + if err != nil { + return nil, fmt.Errorf("failed to get Java dependencies: %w", err) + } + return extractJavaDependencies(out), nil + + case "python": + out, err := exec.Command("pip", "freeze").Output() + if err != nil { + logger.Info("Error getting Python dependencies with `pip` command, trying `pip3` command") + out, err = exec.Command("pip3", "freeze").Output() + if err != nil { + return nil, fmt.Errorf("failed to get Python dependencies: %w", err) + } + } + + return extractDependencies(out), nil + + case "typescript", "javascript": + cmd := exec.Command("sh", "-c", "npm list --depth=0 --parseable | sed 's|.*/||'") + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to get JavaScript/TypeScript dependencies: %w", err) + } + return extractDependencies(out), nil + + default: + return nil, fmt.Errorf("unsupported language: %s", language) + } +} + +func extractJavaDependencies(output []byte) []string { + lines := strings.Split(string(output), "\n") + var dependencies []string + inDependencySection := false + + depRegex := regexp.MustCompile(`^\[INFO\]\s+[\+\|\\\-]{1,2}\s+([\w\.\-]+:[\w\.\-]+):jar:([\w\.\-]+):([\w\.\-]+)`) + + for _, line := range lines { + cleanedLine := strings.TrimSpace(line) + if strings.HasPrefix(cleanedLine, "[INFO]") { + cleanedLine = "[INFO]" + strings.TrimSpace(cleanedLine[6:]) + } + if strings.Contains(cleanedLine, "maven-dependency-plugin") && strings.Contains(cleanedLine, ":list") { + inDependencySection = true + continue + } + + if inDependencySection && (strings.Contains(cleanedLine, "BUILD SUCCESS") || strings.Contains(cleanedLine, "---")) { + inDependencySection = false + continue + } + + if inDependencySection && strings.HasPrefix(cleanedLine, "[INFO]") { + matches := depRegex.FindStringSubmatch(cleanedLine) + if len(matches) >= 4 { + groupArtifact := matches[1] + version := matches[2] + dep := fmt.Sprintf("%s:%s", groupArtifact, version) + dependencies = append(dependencies, dep) + } else { + cleanedLine = strings.TrimPrefix(cleanedLine, "[INFO]") + cleanedLine = strings.TrimSpace(cleanedLine) + + cleanedLine = strings.TrimPrefix(cleanedLine, "+-") + cleanedLine = strings.TrimPrefix(cleanedLine, "\\-") + cleanedLine = strings.TrimPrefix(cleanedLine, "|") + + cleanedLine = strings.TrimSpace(cleanedLine) + + depParts := strings.Split(cleanedLine, ":") + if len(depParts) >= 5 { + dep := fmt.Sprintf("%s:%s:%s", depParts[0], depParts[1], depParts[3]) + dependencies = append(dependencies, dep) + } + } + } + } + return dependencies +} + +func extractDependencies(output []byte) []string { + lines := strings.Split(string(output), "\n") + var dependencies []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + dependencies = append(dependencies, trimmed) + } + } + return dependencies +} + +func isPackageInList(installedPackages []string, packageName string) bool { + for _, pkg := range installedPackages { + if pkg == packageName { + return true + } + } + return false +} + +func installLibraries(libraryCommands string, installedPackages []string, logger *zap.Logger) ([]string, error) { + var newInstalledPackages []string + libraryCommands = strings.TrimSpace(libraryCommands) + if libraryCommands == "" || libraryCommands == "\"\"" { + return newInstalledPackages, nil + } + + commands := strings.Split(libraryCommands, "\n") + for _, command := range commands { + packageName := extractPackageName(command) + + if isPackageInList(installedPackages, packageName) { + continue + } + + _, _, exitCode, _, err := RunCommand(command, "", logger) + if exitCode != 0 || err != nil { + return newInstalledPackages, fmt.Errorf("failed to install library: %s", command) + } + + installedPackages = append(installedPackages, packageName) + newInstalledPackages = append(newInstalledPackages, packageName) + } + return newInstalledPackages, nil +} + +func extractPackageName(command string) string { + fields := strings.Fields(command) + if len(fields) < 3 { + return "" + } + return fields[2] +} + +func uninstallLibraries(language string, installedPackages []string, logger *zap.Logger) error { + for _, command := range installedPackages { + logger.Info(fmt.Sprintf("Uninstalling library: %s", command)) + + var uninstallCommand string + switch strings.ToLower(language) { + case "go": + uninstallCommand = fmt.Sprintf("go mod edit -droprequire %s && go mod tidy", command) + case "python": + uninstallCommand = fmt.Sprintf("pip uninstall -y %s", command) + case "javascript": + uninstallCommand = fmt.Sprintf("npm uninstall %s", command) + case "java": + uninstallCommand = fmt.Sprintf("mvn dependency:purge-local-repository -DreResolve=false -Dinclude=%s", command) + } + if uninstallCommand != "" { + logger.Info(fmt.Sprintf("Uninstalling library with command: %s", uninstallCommand)) + _, _, exitCode, _, err := RunCommand(uninstallCommand, "", logger) + if exitCode != 0 || err != nil { + logger.Warn(fmt.Sprintf("Failed to uninstall library: %s", uninstallCommand), zap.Error(err)) + } + } + } + return nil +} diff --git a/pkg/service/utgen/prompt.go b/pkg/service/utgen/prompt.go index dd232dc3ae..a31bee5841 100644 --- a/pkg/service/utgen/prompt.go +++ b/pkg/service/utgen/prompt.go @@ -5,7 +5,6 @@ import ( "fmt" "html/template" "os" - "path/filepath" "strings" settings "go.keploy.io/server/v2/pkg/service/utgen/assets" @@ -58,15 +57,16 @@ type PromptBuilder struct { Language string Logger *zap.Logger AdditionalPrompt string + InstalledPackages []string } func NewPromptBuilder(srcPath, testPath, covReportContent, includedFiles, additionalInstructions, language, additionalPrompt string, logger *zap.Logger) (*PromptBuilder, error) { var err error src := &Source{ - Name: filepath.Base(srcPath), + Name: srcPath, } test := &Test{ - Name: filepath.Base(testPath), + Name: testPath, } promptBuilder := &PromptBuilder{ Src: src, @@ -141,6 +141,7 @@ func (pb *PromptBuilder) BuildPrompt(file, failedTestRuns string) (*Prompt, erro "language": pb.Language, "max_tests": MAX_TESTS_PER_RUN, "additional_command": pb.AdditionalPrompt, + "installed_packages": formatInstalledPackages(pb.InstalledPackages), } settings := settings.GetSettings() @@ -165,6 +166,14 @@ func (pb *PromptBuilder) BuildPrompt(file, failedTestRuns string) (*Prompt, erro return prompt, nil } +func formatInstalledPackages(packages []string) string { + var sb strings.Builder + for _, pkg := range packages { + sb.WriteString(fmt.Sprintf("- %s\n", pkg)) + } + return sb.String() +} + func renderTemplate(templateText string, variables map[string]interface{}) (string, error) { funcMap := template.FuncMap{ "trim": strings.TrimSpace, diff --git a/utils/signal_others.go b/utils/signal_others.go index e450719c76..1152927ae7 100644 --- a/utils/signal_others.go +++ b/utils/signal_others.go @@ -29,15 +29,15 @@ func SendSignal(logger *zap.Logger, pid int, sig syscall.Signal) error { func ExecuteCommand(ctx context.Context, logger *zap.Logger, userCmd string, cancel func(cmd *exec.Cmd) func() error, waitDelay time.Duration) CmdError { // Run the app as the user who invoked sudo - username := os.Getenv("SUDO_USER") + // username := os.Getenv("SUDO_USER") cmd := exec.CommandContext(ctx, "sh", "-c", userCmd) - if username != "" { - // print all environment variables - logger.Debug("env inherited from the cmd", zap.Any("env", os.Environ())) - // Run the command as the user who invoked sudo to preserve the user environment variables and PATH - cmd = exec.CommandContext(ctx, "sudo", "-E", "-u", os.Getenv("SUDO_USER"), "env", "PATH="+os.Getenv("PATH"), "sh", "-c", userCmd) - } + // if username != "" { + // // print all environment variables + // logger.Debug("env inherited from the cmd", zap.Any("env", os.Environ())) + // // Run the command as the user who invoked sudo to preserve the user environment variables and PATH + // cmd = exec.CommandContext(ctx, "sudo", "-E", "-u", os.Getenv("SUDO_USER"), "env", "PATH="+os.Getenv("PATH"), "sh", "-c", userCmd) + // } // Set the cancel function for the command cmd.Cancel = cancel(cmd) diff --git a/utils/utils.go b/utils/utils.go index 04a29ec606..6e9747b4d3 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "debug/elf" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -26,6 +27,7 @@ import ( "golang.org/x/text/language" "github.com/getsentry/sentry-go" + "github.com/google/uuid" netLib "github.com/shirou/gopsutil/v3/net" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -193,6 +195,16 @@ func DeleteFileIfNotExists(logger *zap.Logger, name string) (err error) { return nil } +func GenerateID() uint64 { + // Random AppId uint64 will be generated and maintain in a map and return the id to client + newUUID := uuid.New() + + // app id will be sent by the client. + // Convert the first 8 bytes of the UUID to an int64 + id := int64(binary.BigEndian.Uint64(newUUID[:8])) + return uint64(id) +} + type GitHubRelease struct { TagName string `json:"tag_name"` Body string `json:"body"`