diff --git a/.github/workflows/autobahn__build-target-test.yml b/.github/workflows/autobahn__build-target-test.yml new file mode 100644 index 0000000000..c20c5d7e40 --- /dev/null +++ b/.github/workflows/autobahn__build-target-test.yml @@ -0,0 +1,228 @@ +name: "autobahn: build/target-tests" + +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened, labeled] + +jobs: + build_autobahn: + # Run on push to master or if PR has 'websocket' label + if: contains(github.event.pull_request.labels.*.name, 'autobahn') || github.event_name == 'push' + name: Build + strategy: + matrix: + #idf_ver: ["release-v5.0", "release-v5.1", "release-v5.2", "release-v5.3", "latest"] + idf_ver: [ "latest"] + idf_target: ["esp32"] + runs-on: ubuntu-22.04 + container: espressif/idf:${{ matrix.idf_ver }} + env: + TEST_DIR: components/esp_websocket_client/examples/autobahn-testsuite/testee + steps: + - name: Checkout esp-protocols + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Build autobahn testee with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} + working-directory: ${{ env.TEST_DIR }} + env: + IDF_TARGET: ${{ matrix.idf_target }} + shell: bash + run: | + . ${IDF_PATH}/export.sh + test -f sdkconfig.ci.plain_tcp && cat sdkconfig.ci.plain_tcp >> sdkconfig.defaults || echo "No sdkconfig.ci.plain_tcp" + idf.py set-target ${{ matrix.idf_target }} + idf.py build + - name: Merge binaries with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} + working-directory: ${{ env.TEST_DIR }}/build + env: + IDF_TARGET: ${{ matrix.idf_target }} + shell: bash + run: | + . ${IDF_PATH}/export.sh + esptool.py --chip ${{ matrix.idf_target }} merge_bin --fill-flash-size 4MB -o flash_image.bin @flash_args + - uses: actions/upload-artifact@v4 + with: + name: autobahn_testee_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }} + path: | + ${{ env.TEST_DIR }}/build/bootloader/bootloader.bin + ${{ env.TEST_DIR }}/build/partition_table/partition-table.bin + ${{ env.TEST_DIR }}/build/*.bin + ${{ env.TEST_DIR }}/build/*.elf + ${{ env.TEST_DIR }}/build/flasher_args.json + ${{ env.TEST_DIR }}/build/config/sdkconfig.h + ${{ env.TEST_DIR }}/build/config/sdkconfig.json + if-no-files-found: error + + run-target-autobahn: + # Skip running on forks since it won't have access to secrets + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'autobahn') || github.event_name == 'push' ) + name: Target test + needs: build_autobahn + strategy: + fail-fast: false + matrix: + idf_ver: ["latest"] + idf_target: ["esp32"] + runs-on: + - self-hosted + - ESP32-ETHERNET-KIT + env: + TEST_DIR: components/esp_websocket_client/examples/autobahn-testsuite + TESTEE_DIR: components/esp_websocket_client/examples/autobahn-testsuite/testee + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/download-artifact@v4 + with: + name: autobahn_testee_bin_${{ matrix.idf_target }}_${{ matrix.idf_ver }} + path: ${{ env.TESTEE_DIR }}/build + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose-plugin || sudo apt-get install -y docker-compose + # Ensure user has permission to use Docker (if not already in docker group) + sudo usermod -aG docker $USER || true + # Start Docker service if not running + sudo systemctl start docker || true + - name: Start Autobahn Fuzzing Server + working-directory: ${{ env.TEST_DIR }} + run: | + # Get host IP address for ESP32 to connect to + HOST_IP=$(hostname -I | awk '{print $1}') + echo "HOST_IP=$HOST_IP" >> $GITHUB_ENV + echo "Autobahn server will be accessible at ws://$HOST_IP:9001" + + # Start the fuzzing server using pre-built image + # For CI, we may need to specify platform if architecture differs + echo "Starting Autobahn fuzzing server..." + # Set platform for CI if needed (uncomment if you get exec format error) + # export DOCKER_DEFAULT_PLATFORM=linux/amd64 + docker compose up -d || docker-compose up -d + + # Wait for server to be ready + echo "Waiting for fuzzing server to start..." + sleep 10 + + # Check if container is running and healthy + if ! docker ps | grep -q ws-fuzzing-server; then + echo "Error: Fuzzing server failed to start" + echo "Container logs:" + docker compose logs || docker-compose logs + echo "Checking available Python executables in container:" + docker compose run --rm fuzzing-server which python python3 || true + exit 1 + fi + + # Verify the server is actually responding + echo "Checking if server is responding..." + sleep 5 + if ! curl -s http://localhost:8080 > /dev/null 2>&1; then + echo "Warning: Server may not be fully ready, but container is running" + docker compose logs --tail=20 || docker-compose logs --tail=20 + fi + + echo "βœ“ Fuzzing server started successfully" + - name: Flash ESP32 Testee + working-directory: ${{ env.TESTEE_DIR }}/build + env: + IDF_TARGET: ${{ matrix.idf_target }} + run: | + python -m esptool --chip ${{ matrix.idf_target }} write_flash 0x0 flash_image.bin + - name: Run Autobahn Tests + working-directory: ${{ env.TESTEE_DIR }} + env: + PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" + run: | + # Detect ESP32 port if not set in environment + if [ -z "${ESP_PORT:-}" ]; then + for port in /dev/ttyUSB* /dev/ttyACM*; do + if [ -e "$port" ]; then + export ESP_PORT="$port" + echo "Detected ESP32 port: $ESP_PORT" + break + fi + done + fi + + # Default to /dev/ttyUSB0 if still not found + export ESP_PORT="${ESP_PORT:-/dev/ttyUSB0}" + + if [ ! -e "$ESP_PORT" ]; then + echo "Error: ESP32 port not found. Please set ESP_PORT environment variable." + echo "Available ports:" + ls -la /dev/tty* || true + exit 1 + fi + + echo "Using ESP32 port: $ESP_PORT" + export PYENV_ROOT="$HOME/.pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init --path)" + eval "$(pyenv init -)" + if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then + echo "Installing Python 3.12.6..." + pyenv install -s 3.12.6 + fi + if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then + echo "Creating pyenv virtualenv 'myenv'..." + pyenv virtualenv 3.12.6 myenv + fi + pyenv activate myenv + python --version + pip install --prefer-binary pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code esptool pyserial + pip install --extra-index-url https://dl.espressif.com/pypi/ -r $GITHUB_WORKSPACE/ci/requirements.txt + + echo "Starting Autobahn test suite on ESP32..." + echo "Tests may take 15-30 minutes to complete..." + + # Send server URI via serial (stdin) and monitor for completion + # Script is in the parent directory (TEST_DIR) from TESTEE_DIR + SERVER_URI="ws://$HOST_IP:9001" + echo "Sending server URI to ESP32: $SERVER_URI" + python3 ../scripts/monitor_serial.py --port "$ESP_PORT" --uri "$SERVER_URI" --timeout 2400 + - name: Collect Test Reports + working-directory: ${{ env.TEST_DIR }} + if: always() + run: | + # Stop the fuzzing server + docker compose down || docker-compose down + + # Check if reports were generated + if [ -d "reports/clients" ]; then + echo "βœ“ Test reports found" + ls -la reports/clients/ + else + echo "⚠ No test reports found in reports/clients/" + fi + - name: Generate Test Summary + working-directory: ${{ env.TEST_DIR }} + if: always() + run: | + # Generate summary from test results + # Check for JSON files in both reports/ and reports/clients/ + if [ -d "reports" ] && ( [ -n "$(ls -A reports/*.json 2>/dev/null)" ] || [ -n "$(ls -A reports/clients/*.json 2>/dev/null)" ] ); then + echo "Generating test summary..." + python3 scripts/generate_summary.py + echo "" + echo "Summary generated successfully!" + if [ -f "reports/summary.html" ]; then + echo "HTML summary available at: reports/summary.html" + fi + else + echo "⚠ No JSON test results found, skipping summary generation" + fi + - uses: actions/upload-artifact@v4 + if: always() + with: + name: autobahn_reports_${{ matrix.idf_target }}_${{ matrix.idf_ver }} + path: | + ${{ env.TEST_DIR }}/reports/** + if-no-files-found: warn + diff --git a/components/esp_websocket_client/esp_websocket_client.c b/components/esp_websocket_client/esp_websocket_client.c index 21a172d91e..807398b654 100644 --- a/components/esp_websocket_client/esp_websocket_client.c +++ b/components/esp_websocket_client/esp_websocket_client.c @@ -241,9 +241,29 @@ static esp_err_t esp_websocket_client_dispatch_event(esp_websocket_client_handle return esp_event_loop_run(client->event_handle, 0); } +/** + * @brief Abort the WebSocket connection and initiate reconnection or shutdown + * + * @param client WebSocket client handle + * @param error_type Type of error that caused the abort + * + * @return ESP_OK on success, ESP_FAIL on failure + * + * @note PRECONDITION: client->lock MUST be held by the calling thread before calling this function. + * This function does NOT acquire the lock itself. Calling without the lock will result in + * race conditions and undefined behavior. + */ static esp_err_t esp_websocket_client_abort_connection(esp_websocket_client_handle_t client, esp_websocket_error_type_t error_type) { ESP_WS_CLIENT_STATE_CHECK(TAG, client, return ESP_FAIL); + + + if (client->state == WEBSOCKET_STATE_CLOSING || client->state == WEBSOCKET_STATE_UNKNOW || + client->state == WEBSOCKET_STATE_WAIT_TIMEOUT) { + ESP_LOGW(TAG, "Connection already closing/closed, skipping abort"); + return ESP_OK; + } + esp_transport_close(client->transport); if (!client->config->auto_reconnect) { @@ -256,6 +276,17 @@ static esp_err_t esp_websocket_client_abort_connection(esp_websocket_client_hand } client->error_handle.error_type = error_type; esp_websocket_client_dispatch_event(client, WEBSOCKET_EVENT_DISCONNECTED, NULL, 0); + + if (client->errormsg_buffer) { + ESP_LOGD(TAG, "Freeing error buffer (%d bytes) - Free heap: %" PRIu32 " bytes", + client->errormsg_size, esp_get_free_heap_size()); + free(client->errormsg_buffer); + client->errormsg_buffer = NULL; + client->errormsg_size = 0; + } else { + ESP_LOGD(TAG, "Disconnect - Free heap: %" PRIu32 " bytes", esp_get_free_heap_size()); + } + return ESP_OK; } @@ -453,6 +484,8 @@ static void destroy_and_free_resources(esp_websocket_client_handle_t client) esp_websocket_client_destroy_config(client); if (client->transport_list) { esp_transport_list_destroy(client->transport_list); + client->transport_list = NULL; + client->transport = NULL; } vSemaphoreDelete(client->lock); #ifdef CONFIG_ESP_WS_CLIENT_SEPARATE_TX_LOCK @@ -679,8 +712,18 @@ static int esp_websocket_client_send_with_exact_opcode(esp_websocket_client_hand } else { esp_websocket_client_error(client, "esp_transport_write() returned %d, errno=%d", ret, errno); } + ESP_LOGD(TAG, "Calling abort_connection due to send error"); +#ifdef CONFIG_ESP_WS_CLIENT_SEPARATE_TX_LOCK + xSemaphoreGiveRecursive(client->tx_lock); + xSemaphoreTakeRecursive(client->lock, portMAX_DELAY); + esp_websocket_client_abort_connection(client, WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT); + xSemaphoreGiveRecursive(client->lock); + return ret; +#else + // Already holding client->lock, safe to call esp_websocket_client_abort_connection(client, WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT); goto unlock_and_return; +#endif } opcode = 0; widx += wlen; @@ -1019,7 +1062,6 @@ static esp_err_t esp_websocket_client_recv(esp_websocket_client_handle_t client) esp_websocket_free_buf(client, false); return ESP_OK; } - esp_websocket_client_dispatch_event(client, WEBSOCKET_EVENT_DATA, client->rx_buffer, rlen); client->payload_offset += rlen; @@ -1030,15 +1072,35 @@ static esp_err_t esp_websocket_client_recv(esp_websocket_client_handle_t client) const char *data = (client->payload_len == 0) ? NULL : client->rx_buffer; ESP_LOGD(TAG, "Sending PONG with payload len=%d", client->payload_len); #ifdef CONFIG_ESP_WS_CLIENT_SEPARATE_TX_LOCK + xSemaphoreGiveRecursive(client->lock); // Release client->lock + + // Now acquire tx_lock with timeout (consistent with PING/CLOSE handling) if (xSemaphoreTakeRecursive(client->tx_lock, WEBSOCKET_TX_LOCK_TIMEOUT_MS) != pdPASS) { - ESP_LOGE(TAG, "Could not lock ws-client within %d timeout", WEBSOCKET_TX_LOCK_TIMEOUT_MS); - return ESP_FAIL; + ESP_LOGE(TAG, "Could not lock ws-client within %d timeout for PONG", WEBSOCKET_TX_LOCK_TIMEOUT_MS); + xSemaphoreTakeRecursive(client->lock, portMAX_DELAY); // Re-acquire client->lock before returning + esp_websocket_free_buf(client, false); // Free rx_buffer to prevent memory leak + return ESP_OK; // Return gracefully, caller expects client->lock to be held } -#endif + + // Re-acquire client->lock to maintain consistency + xSemaphoreTakeRecursive(client->lock, portMAX_DELAY); + + + // Another thread may have closed it while we didn't hold client->lock + if (client->state == WEBSOCKET_STATE_CLOSING || client->state == WEBSOCKET_STATE_UNKNOW || + client->state == WEBSOCKET_STATE_WAIT_TIMEOUT || client->transport == NULL) { + ESP_LOGW(TAG, "Transport closed while preparing PONG, skipping send"); + xSemaphoreGiveRecursive(client->tx_lock); + esp_websocket_free_buf(client, false); // Free rx_buffer to prevent memory leak + return ESP_OK; // Caller expects client->lock to be held, which it is + } + esp_transport_ws_send_raw(client->transport, WS_TRANSPORT_OPCODES_PONG | WS_TRANSPORT_OPCODES_FIN, data, client->payload_len, client->config->network_timeout_ms); -#ifdef CONFIG_ESP_WS_CLIENT_SEPARATE_TX_LOCK xSemaphoreGiveRecursive(client->tx_lock); +#else + esp_transport_ws_send_raw(client->transport, WS_TRANSPORT_OPCODES_PONG | WS_TRANSPORT_OPCODES_FIN, data, client->payload_len, + client->config->network_timeout_ms); #endif } else if (client->last_opcode == WS_TRANSPORT_OPCODES_PONG) { client->wait_for_pong_resp = false; @@ -1136,7 +1198,20 @@ static void esp_websocket_client_task(void *pv) client->state = WEBSOCKET_STATE_CONNECTED; client->wait_for_pong_resp = false; client->error_handle.error_type = WEBSOCKET_ERROR_TYPE_NONE; + client->payload_len = 0; + client->payload_offset = 0; + client->last_fin = false; + client->last_opcode = WS_TRANSPORT_OPCODES_NONE; + esp_websocket_client_dispatch_event(client, WEBSOCKET_EVENT_CONNECTED, NULL, 0); + // Check for any data that may have arrived during handshake + int immediate_poll = esp_transport_poll_read(client->transport, 0); // Non-blocking + if (immediate_poll >= 0) { + esp_err_t recv_result = esp_websocket_client_recv(client); + if (recv_result == ESP_OK) { + esp_event_loop_run(client->event_handle, 0); + } + } break; case WEBSOCKET_STATE_CONNECTED: if ((CLOSE_FRAME_SENT_BIT & xEventGroupGetBits(client->status_bits)) == 0) { // only send and check for PING @@ -1214,12 +1289,13 @@ static void esp_websocket_client_task(void *pv) esp_websocket_client_abort_connection(client, WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT); xSemaphoreGiveRecursive(client->lock); } else if (read_select > 0) { + xSemaphoreTakeRecursive(client->lock, lock_timeout); if (esp_websocket_client_recv(client) == ESP_FAIL) { ESP_LOGE(TAG, "Error receive data"); - xSemaphoreTakeRecursive(client->lock, lock_timeout); + // Note: Already holding client->lock from line above esp_websocket_client_abort_connection(client, WEBSOCKET_ERROR_TYPE_TCP_TRANSPORT); - xSemaphoreGiveRecursive(client->lock); } + xSemaphoreGiveRecursive(client->lock); } else { ESP_LOGV(TAG, "Read poll timeout: skipping esp_transport_poll_read()."); } diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/.gitignore b/components/esp_websocket_client/examples/autobahn-testsuite/.gitignore new file mode 100644 index 0000000000..8a637e62cc --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/.gitignore @@ -0,0 +1,13 @@ +# Autobahn testsuite generated files +reports/ + +# ESP-IDF build artifacts +testee/build/ +testee/sdkconfig +testee/sdkconfig.old +testee/dependencies.lock + +# Python +__pycache__/ +*.pyc + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/README.md b/components/esp_websocket_client/examples/autobahn-testsuite/README.md new file mode 100644 index 0000000000..6b15190a5f --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/README.md @@ -0,0 +1,275 @@ +# Autobahn WebSocket Testsuite for esp_websocket_client + +This directory contains the setup for testing `esp_websocket_client` against the industry-standard [Autobahn WebSocket Testsuite](https://github.com/crossbario/autobahn-testsuite). + +The Autobahn Testsuite is the de facto standard for testing WebSocket protocol compliance. It runs over 500 test cases covering: +- Frame parsing and generation +- Text and binary messages +- Fragmentation +- Control frames (PING, PONG, CLOSE) +- UTF-8 validation +- Protocol violations +- Edge cases and error handling + +## πŸ“‹ Prerequisites + +1. **Docker** - For running the Autobahn testsuite server + - **Apple Silicon Macs (M1/M2/M3)**: The image runs via Rosetta 2 emulation (already configured) + - **Intel Macs / Linux**: Native support +2. **ESP32 device** with WiFi capability +3. **ESP-IDF** development environment +4. **Network** - ESP32 and Docker host on the same network + +## πŸš€ Quick Start + +### Step 1: Start the Autobahn Fuzzing Server + +```bash +cd autobahn-testsuite +docker-compose up +``` + +This will: +- Start the Autobahn fuzzing server on port 9001 +- Start a web server on port 8080 for viewing reports +- Mount the `reports/` directory for test results + +You should see output like: +``` +Autobahn WebSockets 0.7.4/0.10.9 Fuzzing Server (Port 9001) +Ok, will run 521 test cases for any clients connecting +``` + +### Step 2: Configure the ESP32 Testee Client + +1. Find your Docker host IP address: + ```bash + # On Linux/Mac + ifconfig + # or + ip addr show + + # Look for your local network IP (e.g., 192.168.1.100) + ``` + +2. Update the configuration: + ```bash + cd testee + idf.py menuconfig + ``` + +3. Configure: + - **Example Connection Configuration** β†’ Set your WiFi SSID and password + - **Autobahn Testsuite Configuration** β†’ Set the server URI (e.g., `ws://192.168.1.100:9001`) + + Or edit `sdkconfig.defaults` directly: + ``` + CONFIG_EXAMPLE_WIFI_SSID="YourWiFiSSID" + CONFIG_EXAMPLE_WIFI_PASSWORD="YourWiFiPassword" + CONFIG_AUTOBAHN_SERVER_URI="ws://192.168.1.100:9001" + ``` + +### Step 3: Build and Flash the Testee + +```bash +cd testee + +# Build +idf.py build + +# Flash and monitor +idf.py -p /dev/ttyUSB0 flash monitor +``` + +Replace `/dev/ttyUSB0` with your ESP32's serial port. + +### Step 4: Watch the Tests Run + +The ESP32 will: +1. Connect to WiFi +2. Query the fuzzing server for the number of test cases +3. Run each test case sequentially +4. Echo back all received messages (as required by the testsuite) +5. Generate a final report + +You'll see output like: +``` +I (12345) autobahn_testee: ========== Test Case 1/300 ========== +I (12346) autobahn_testee: Running test case 1: ws://192.168.1.100:9001/runCase?case=1&agent=esp_websocket_client +I (12450) autobahn_testee: WEBSOCKET_EVENT_CONNECTED +... +I (12550) autobahn_testee: Test case 1 completed +``` + +### Step 5: View the Results + +Once all tests complete, view the HTML report: + +1. Open your web browser to: `http://:8080` +2. Click on the generated report +3. You'll see a comprehensive breakdown of all test results: + - βœ… **Pass** - Compliant behavior + - ⚠️ **Non-Strict** - Minor deviation (usually acceptable) + - ❌ **Fail** - Protocol violation + - ℹ️ **Informational** - Notes about behavior + +The report will also be saved in `autobahn-testsuite/reports/clients/` directory. + +## πŸ“‚ Directory Structure + +``` +autobahn-testsuite/ +β”œβ”€β”€ docker-compose.yml # Docker configuration +β”œβ”€β”€ config/ +β”‚ └── fuzzingserver.json # Testsuite server configuration +β”œβ”€β”€ reports/ # Generated test reports (created automatically) +β”‚ └── clients/ +β”‚ └── index.html # Main report page +β”œβ”€β”€ testee/ # ESP32 testee client project +β”‚ β”œβ”€β”€ CMakeLists.txt +β”‚ β”œβ”€β”€ sdkconfig.defaults +β”‚ └── main/ +β”‚ β”œβ”€β”€ autobahn_testee.c # Main testee implementation +β”‚ β”œβ”€β”€ CMakeLists.txt +β”‚ └── Kconfig.projbuild +└── README.md # This file +``` + +## βš™οΈ Configuration + +### Fuzzing Server Configuration + +Edit `config/fuzzingserver.json` to customize test behavior: + +```json +{ + "url": "ws://0.0.0.0:9001", + "outdir": "/reports", + "cases": ["*"], + "exclude-cases": [ + "9.*", // Excludes performance/mass tests + "12.*", // Excludes compression tests (permessage-deflate) + "13.*" // Excludes compression tests + ] +} +``` + +**Note**: Test cases 12.* and 13.* are excluded by default as they test WebSocket compression (RFC7692), which is typically not implemented in embedded clients. + +### Test Case Categories + +- **1.*** - Framing +- **2.*** - Pings/Pongs +- **3.*** - Reserved Bits +- **4.*** - Opcodes +- **5.*** - Fragmentation +- **6.*** - UTF-8 Handling +- **7.*** - Close Handling +- **9.*** - Performance & Limits (excluded by default) +- **10.*** - Miscellaneous +- **12.*** - WebSocket Compression (excluded by default) +- **13.*** - WebSocket Compression (excluded by default) + +## πŸ”§ Troubleshooting + +### Apple Silicon Mac (M1/M2/M3) Platform Warning + +If you see a warning like: +``` +The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) +``` + +**This is normal and expected!** The Autobahn testsuite image only supports amd64, but Docker Desktop for Mac will automatically run it through Rosetta 2 emulation. The `platform: linux/amd64` line in `docker-compose.yml` handles this. + +**Note**: There may be a slight performance overhead due to emulation, but it's typically not noticeable for this use case. + +### ESP32 Can't Connect to Server + +1. Verify Docker host IP is correct: + ```bash + docker inspect ws-fuzzing-server | grep IPAddress + ``` + +2. Check firewall rules allow connections on port 9001 + +3. Ensure ESP32 and Docker host are on same network + +4. Test connectivity: + ```bash + # From another machine on same network + curl -i -N -H "Connection: Upgrade" -H "Upgrade: websocket" \ + -H "Sec-WebSocket-Key: SGVsbG8sIHdvcmxkIQ==" \ + -H "Sec-WebSocket-Version: 13" \ + http://:9001/getCaseCount + ``` + +### Tests Timeout or Hang + +1. Increase network timeout in `autobahn_testee.c`: + ```c + websocket_cfg.network_timeout_ms = 30000; // 30 seconds + ``` + +2. Check WiFi signal strength + +3. Monitor memory usage - insufficient heap can cause issues + +### Some Tests Fail + +This is normal! The testsuite is very strict and tests edge cases. Common issues: + +- **UTF-8 validation** - May fail if not strictly validating text frames +- **Close frame handling** - May fail if close reason not properly echoed +- **Reserved bits** - May fail if not properly validating reserved bits + +Review the HTML report to understand specific failures and determine if they're critical. + +## πŸ“Š Understanding Results + +### Result Categories + +- **Passed**: Implementation is compliant +- **Non-Strict**: Minor deviation, usually acceptable +- **Failed**: Protocol violation detected +- **Informational**: Observation about behavior + +### Common Issues + +1. **Text frame UTF-8 validation** + - Testsuite sends invalid UTF-8 in text frames + - Client should reject these + +2. **Close frame payload** + - Close frames can have a status code + reason + - Client should echo back close frames properly + +3. **Fragmentation** + - Tests various fragmentation scenarios + - Client must properly handle continuation frames + +## 🎯 Goal + +A high-quality WebSocket implementation should: +- Pass all core protocol tests (1.* through 11.*) +- Have minimal "Non-Strict" results +- Have no "Failed" results in critical areas + +## πŸ“š References + +- [Autobahn Testsuite Documentation](https://crossbar.io/autobahn/) +- [RFC 6455 - The WebSocket Protocol](https://tools.ietf.org/html/rfc6455) +- [Autobahn Testsuite GitHub](https://github.com/crossbario/autobahn-testsuite) + +## πŸ› Reporting Issues + +If you find protocol compliance issues with `esp_websocket_client`, please report them with: +1. The specific test case number that failed +2. The test report HTML output +3. ESP32 logs during the test +4. Your configuration (sdkconfig) + +## πŸ“ License + +This testsuite setup is provided under the same license as esp_websocket_client. +The Autobahn Testsuite itself is licensed under Apache 2.0. + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/config/fuzzingserver-quick.json b/components/esp_websocket_client/examples/autobahn-testsuite/config/fuzzingserver-quick.json new file mode 100644 index 0000000000..f2cf4a99bd --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/config/fuzzingserver-quick.json @@ -0,0 +1,13 @@ +{ + "url": "ws://0.0.0.0:9001", + "options": { + "failByDrop": false + }, + "outdir": "/reports", + "webport": 8080, + "cases": ["1.*", "2.*", "3.*", "4.*", "5.*", "7.*", "10.*"], + "exclude-cases": [], + "exclude-agent-cases": {}, + "_comment": "Quick test config - runs ~150 core tests, excludes UTF-8 (6.*), performance (9.*), and compression (12.*, 13.*)" +} + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/config/fuzzingserver.json b/components/esp_websocket_client/examples/autobahn-testsuite/config/fuzzingserver.json new file mode 100644 index 0000000000..241ae86f61 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/config/fuzzingserver.json @@ -0,0 +1,16 @@ +{ + "url": "ws://0.0.0.0:9001", + "options": { + "failByDrop": false + }, + "outdir": "/reports", + "webport": 8080, + "cases": ["*"], + "exclude-cases": [ + "9.*", + "12.*", + "13.*" + ], + "exclude-agent-cases": {} +} + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/docker-compose.yml b/components/esp_websocket_client/examples/autobahn-testsuite/docker-compose.yml new file mode 100644 index 0000000000..d3df30a6c2 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/docker-compose.yml @@ -0,0 +1,12 @@ +services: + fuzzing-server: + image: crossbario/autobahn-testsuite:latest + container_name: ws-fuzzing-server + platform: linux/amd64 # <β€” enforce amd64, use QEMU on Apple Silicon + ports: + - "9001:9001" + - "8080:8080" + volumes: + - ./config:/config + - ./reports:/reports + command: wstest -m fuzzingserver -s /config/fuzzingserver.json diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/run_tests.sh b/components/esp_websocket_client/examples/autobahn-testsuite/run_tests.sh new file mode 100755 index 0000000000..c1dad217cb --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/run_tests.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Autobahn Testsuite Runner Script +# This script automates the process of running WebSocket protocol compliance tests + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}======================================${NC}" +echo -e "${BLUE}Autobahn WebSocket Testsuite Runner${NC}" +echo -e "${BLUE}======================================${NC}" +echo + +# Check if Docker is running +if ! docker info > /dev/null 2>&1; then + echo -e "${RED}Error: Docker is not running${NC}" + echo "Please start Docker and try again" + exit 1 +fi + +# Function to get host IP +get_host_ip() { + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -1 + else + # Linux + hostname -I | awk '{print $1}' + fi +} + +HOST_IP=$(get_host_ip) + +echo -e "${GREEN}Step 1: Starting Autobahn Fuzzing Server${NC}" +echo "Host IP detected: $HOST_IP" +echo + +# Start the fuzzing server +docker-compose up -d + +# Wait for server to be ready +echo "Waiting for fuzzing server to start..." +sleep 5 + +# Check if container is running +if ! docker ps | grep -q ws-fuzzing-server; then + echo -e "${RED}Error: Fuzzing server failed to start${NC}" + docker-compose logs + exit 1 +fi + +echo -e "${GREEN}βœ“ Fuzzing server started successfully${NC}" +echo " WebSocket endpoint: ws://$HOST_IP:9001" +echo " Web interface: http://$HOST_IP:8080" +echo + +echo -e "${YELLOW}Step 2: Configure and Flash ESP32${NC}" +echo +echo "Before running the testee client on ESP32:" +echo " 1. Update WiFi credentials in testee/sdkconfig.defaults" +echo " 2. Update Autobahn server URI to: ws://$HOST_IP:9001" +echo " 3. Build and flash:" +echo " cd testee" +echo " idf.py build" +echo " idf.py -p PORT flash monitor" +echo +echo -e "${YELLOW}Press Enter when ESP32 testing is complete...${NC}" +read + +echo +echo -e "${GREEN}Step 3: Viewing Results${NC}" +echo + +# Open the report in browser +if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + open "http://localhost:8080" +elif command -v xdg-open > /dev/null; then + # Linux + xdg-open "http://localhost:8080" +else + echo "Open http://localhost:8080 in your browser to view results" +fi + +echo +echo -e "${BLUE}Test reports are available at:${NC}" +echo " Browser: http://localhost:8080" +echo " Directory: $SCRIPT_DIR/reports/clients/" +echo + +echo -e "${YELLOW}To stop the fuzzing server:${NC}" +echo " docker-compose down" +echo + +echo -e "${GREEN}Testing complete!${NC}" + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/scripts/analyze_results.py b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/analyze_results.py new file mode 100755 index 0000000000..9e79ce5110 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/analyze_results.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +Autobahn Test Results Analyzer + +Parses the JSON report from Autobahn testsuite and provides a summary. +""" + +import json +import sys +from pathlib import Path +from collections import defaultdict + +def analyze_results(report_path): + """Analyze the Autobahn test results JSON file.""" + + with open(report_path, 'r') as f: + data = json.load(f) + + # Find the agent name (should be esp_websocket_client) + agents = list(data.keys()) + if not agents: + print("❌ No test results found in report") + return 1 + + agent = agents[0] + results = data[agent] + + print(f"\n{'='*60}") + print(f"Autobahn WebSocket Testsuite Results") + print(f"Agent: {agent}") + print(f"{'='*60}\n") + + # Count results by behavior + counts = defaultdict(int) + by_category = defaultdict(lambda: defaultdict(int)) + failures = [] + + for case_id, result in results.items(): + behavior = result.get('behavior', 'UNKNOWN') + counts[behavior] += 1 + + # Extract category (e.g., "1" from "1.2.3") + category = case_id.split('.')[0] + by_category[category][behavior] += 1 + + if behavior in ['FAILED', 'UNIMPLEMENTED']: + failures.append({ + 'case': case_id, + 'behavior': behavior, + 'description': result.get('description', 'N/A'), + 'result': result.get('behaviorClose', 'N/A') + }) + + # Print overall summary + total = sum(counts.values()) + print(f"Overall Results ({total} tests):") + print(f" βœ… PASSED: {counts['OK']:3d} ({counts['OK']/total*100:5.1f}%)") + print(f" ⚠️ NON-STRICT: {counts['NON-STRICT']:3d} ({counts['NON-STRICT']/total*100:5.1f}%)") + print(f" ℹ️ INFORMATIONAL: {counts['INFORMATIONAL']:3d} ({counts['INFORMATIONAL']/total*100:5.1f}%)") + print(f" ❌ FAILED: {counts['FAILED']:3d} ({counts['FAILED']/total*100:5.1f}%)") + print(f" β­• UNIMPLEMENTED: {counts['UNIMPLEMENTED']:3d} ({counts['UNIMPLEMENTED']/total*100:5.1f}%)") + print() + + # Print by category + print(f"Results by Category:") + print(f" {'Cat':<4} {'Pass':>6} {'N-S':>6} {'Info':>6} {'Fail':>6} {'N/I':>6} {'Total':>6}") + print(f" {'-'*4} {'-'*6} {'-'*6} {'-'*6} {'-'*6} {'-'*6} {'-'*6}") + + for category in sorted(by_category.keys(), key=lambda x: int(x) if x.isdigit() else 999): + cat_results = by_category[category] + cat_total = sum(cat_results.values()) + print(f" {category:>3}. " + f"{cat_results['OK']:>6} " + f"{cat_results['NON-STRICT']:>6} " + f"{cat_results['INFORMATIONAL']:>6} " + f"{cat_results['FAILED']:>6} " + f"{cat_results['UNIMPLEMENTED']:>6} " + f"{cat_total:>6}") + + print() + + # Print failures details + if failures: + print(f"\n❌ Failed/Unimplemented Tests ({len(failures)}):") + print(f" {'-'*60}") + for fail in failures: + print(f" Case {fail['case']}: {fail['behavior']}") + print(f" Description: {fail['description'][:70]}") + print(f" Result: {fail['result']}") + print() + else: + print("\nπŸŽ‰ All tests passed or acceptable! No failures.") + + # Quality assessment + print(f"\n{'='*60}") + print("Quality Assessment:") + + pass_rate = (counts['OK'] + counts['NON-STRICT']) / total * 100 + fail_rate = counts['FAILED'] / total * 100 + + if fail_rate == 0 and counts['OK'] > total * 0.8: + print(" 🌟 EXCELLENT - Production ready!") + elif fail_rate < 5 and pass_rate > 85: + print(" βœ… GOOD - Minor issues to address") + elif fail_rate < 10 and pass_rate > 70: + print(" ⚠️ FAIR - Several issues need fixing") + else: + print(" ❌ POOR - Significant protocol compliance issues") + + print(f"{'='*60}\n") + + return 0 if fail_rate < 5 else 1 + +def main(): + if len(sys.argv) > 1: + report_path = Path(sys.argv[1]) + else: + # Default location + script_dir = Path(__file__).parent.parent + report_path = script_dir / "reports" / "clients" / "index.json" + + if not report_path.exists(): + print(f"❌ Error: Report file not found: {report_path}") + print("\nUsage: python analyze_results.py [path/to/index.json]") + print(f"Default: {report_path}") + return 1 + + return analyze_results(report_path) + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/scripts/generate_summary.py b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/generate_summary.py new file mode 100755 index 0000000000..567a16971b --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/generate_summary.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +""" +Autobahn Test Results Summary Generator + +This script parses all JSON test results and generates a comprehensive summary. +""" + +import json +import glob +import os +from collections import defaultdict +from pathlib import Path +from datetime import datetime + +# ANSI color codes for terminal output +class Colors: + GREEN = '\033[92m' + RED = '\033[91m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + CYAN = '\033[96m' + MAGENTA = '\033[95m' + BOLD = '\033[1m' + END = '\033[0m' + +# Test category descriptions +CATEGORIES = { + '1': {'name': 'Framing', 'critical': True, 'description': 'Basic frame structure'}, + '2': {'name': 'Ping/Pong', 'critical': True, 'description': 'Control frames'}, + '3': {'name': 'Reserved Bits', 'critical': True, 'description': 'RSV validation'}, + '4': {'name': 'Opcodes', 'critical': True, 'description': 'Valid/invalid opcodes'}, + '5': {'name': 'Fragmentation', 'critical': True, 'description': 'Message fragments'}, + '6': {'name': 'UTF-8', 'critical': False, 'description': 'Text validation'}, + '7': {'name': 'Close Handshake', 'critical': True, 'description': 'Connection closing'}, + '9': {'name': 'Performance', 'critical': False, 'description': 'Large messages'}, + '10': {'name': 'Miscellaneous', 'critical': True, 'description': 'Edge cases'}, + '12': {'name': 'Compression', 'critical': False, 'description': 'RFC 7692'}, + '13': {'name': 'Compression', 'critical': False, 'description': 'RFC 7692'}, +} + +def get_category_from_test_id(test_id): + """Extract category number from test ID (e.g., '1.1.1' -> '1')""" + return test_id.split('.')[0] + +def parse_test_results(reports_dir): + """Parse all JSON test results""" + results = [] + # Look for JSON files in reports directory and reports/clients subdirectory + json_files = glob.glob(os.path.join(reports_dir, "*.json")) + json_files.extend(glob.glob(os.path.join(reports_dir, "clients", "*.json"))) + + for json_file in json_files: + try: + with open(json_file, 'r') as f: + data = json.load(f) + results.append({ + 'id': data.get('id', 'unknown'), + 'behavior': data.get('behavior', 'UNKNOWN'), + 'description': data.get('description', ''), + 'duration': data.get('duration', 0), + 'result': data.get('result', ''), + }) + except Exception as e: + print(f"Error parsing {json_file}: {e}") + + # Sort by test ID with proper numeric sorting + def sort_key(result): + parts = result['id'].split('.') + # Convert to sortable format: pad numbers, keep strings + sortable = [] + for p in parts: + if p.isdigit(): + sortable.append((0, int(p))) # numbers sort before strings + else: + sortable.append((1, p)) # strings sort after numbers + return tuple(sortable) + + return sorted(results, key=sort_key) + +def generate_summary(results): + """Generate comprehensive summary statistics""" + + # Overall stats + total = len(results) + by_behavior = defaultdict(int) + by_category = defaultdict(lambda: defaultdict(int)) + + for result in results: + behavior = result['behavior'] + by_behavior[behavior] += 1 + + category = get_category_from_test_id(result['id']) + by_category[category][behavior] += 1 + + return { + 'total': total, + 'by_behavior': dict(by_behavior), + 'by_category': dict(by_category) + } + +def calculate_grade(pass_rate): + """Calculate letter grade based on pass rate""" + if pass_rate >= 90: + return 'A', Colors.GREEN + elif pass_rate >= 80: + return 'B', Colors.GREEN + elif pass_rate >= 70: + return 'C', Colors.YELLOW + elif pass_rate >= 60: + return 'D', Colors.YELLOW + else: + return 'F', Colors.RED + +def print_banner(text): + """Print a banner""" + print() + print(f"{Colors.BLUE}{'=' * 80}{Colors.END}") + print(f"{Colors.BLUE}{Colors.BOLD}{text:^80}{Colors.END}") + print(f"{Colors.BLUE}{'=' * 80}{Colors.END}") + print() + +def print_summary_table(summary): + """Print overall summary table""" + print_banner("OVERALL TEST RESULTS") + + total = summary['total'] + by_behavior = summary['by_behavior'] + + # Calculate percentages + passed = by_behavior.get('OK', 0) + failed = by_behavior.get('FAILED', 0) + non_strict = by_behavior.get('NON-STRICT', 0) + informational = by_behavior.get('INFORMATIONAL', 0) + unimplemented = by_behavior.get('UNIMPLEMENTED', 0) + + pass_rate = (passed / total * 100) if total > 0 else 0 + fail_rate = (failed / total * 100) if total > 0 else 0 + + grade, grade_color = calculate_grade(pass_rate) + + print(f"Total Tests: {Colors.BOLD}{total}{Colors.END}") + print() + print(f"{Colors.GREEN}βœ… PASSED: {passed:3d} ({pass_rate:5.1f}%){Colors.END}") + print(f"{Colors.RED}❌ FAILED: {failed:3d} ({fail_rate:5.1f}%){Colors.END}") + if non_strict > 0: + print(f"{Colors.YELLOW}⚠️ NON-STRICT: {non_strict:3d} ({non_strict/total*100:5.1f}%){Colors.END}") + if informational > 0: + print(f"{Colors.CYAN}ℹ️ INFORMATIONAL: {informational:3d} ({informational/total*100:5.1f}%){Colors.END}") + if unimplemented > 0: + print(f"{Colors.MAGENTA}πŸ”§ UNIMPLEMENTED: {unimplemented:3d} ({unimplemented/total*100:5.1f}%){Colors.END}") + + print() + print(f"Overall Grade: {grade_color}{Colors.BOLD}{grade}{Colors.END}") + print() + + # Embedded client rating + if pass_rate >= 70: + rating = "Excellent" + rating_color = Colors.GREEN + elif pass_rate >= 55: + rating = "Good" + rating_color = Colors.GREEN + elif pass_rate >= 40: + rating = "Acceptable" + rating_color = Colors.YELLOW + else: + rating = "Needs Improvement" + rating_color = Colors.RED + + print(f"Embedded Client Rating: {rating_color}{Colors.BOLD}{rating}{Colors.END}") + print() + +def print_category_breakdown(summary): + """Print detailed category breakdown""" + print_banner("RESULTS BY TEST CATEGORY") + + by_category = summary['by_category'] + + # Header + print(f"{'Category':<25} {'Total':>7} {'Pass':>7} {'Fail':>7} {'Rate':>8} {'Critical':>10} {'Grade':>7}") + print(f"{'-'*25} {'-'*7} {'-'*7} {'-'*7} {'-'*8} {'-'*10} {'-'*7}") + + # Sort categories numerically + sorted_categories = sorted(by_category.keys(), key=lambda x: int(x) if x.isdigit() else 999) + + for cat_num in sorted_categories: + cat_stats = by_category[cat_num] + cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True}) + + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + failed = cat_stats.get('FAILED', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + grade, grade_color = calculate_grade(pass_rate) + + # Format category name + cat_name = f"{cat_num}.* {cat_info['name']}" + critical = "Yes" if cat_info['critical'] else "No" + + # Color code the pass rate + if pass_rate >= 70: + rate_color = Colors.GREEN + elif pass_rate >= 50: + rate_color = Colors.YELLOW + else: + rate_color = Colors.RED + + print(f"{cat_name:<25} {total:>7} {passed:>7} {failed:>7} " + f"{rate_color}{pass_rate:>7.1f}%{Colors.END} {critical:>10} " + f"{grade_color}{grade:>7}{Colors.END}") + + print() + +def print_failed_tests(results, limit=20): + """Print list of failed tests""" + print_banner("FAILED TESTS (First 20)") + + failed = [r for r in results if r['behavior'] == 'FAILED'] + + if not failed: + print(f"{Colors.GREEN}πŸŽ‰ No failed tests!{Colors.END}\n") + return + + print(f"Total Failed: {Colors.RED}{Colors.BOLD}{len(failed)}{Colors.END}\n") + + for i, test in enumerate(failed[:limit], 1): + print(f"{i:2d}. {Colors.RED}Test {test['id']:<12}{Colors.END} - {test['description'][:60]}") + if test['result']: + print(f" Reason: {test['result'][:100]}") + + if len(failed) > limit: + print(f"\n... and {len(failed) - limit} more failed tests") + + print() + +def print_recommendations(summary): + """Print recommendations based on results""" + print_banner("RECOMMENDATIONS") + + by_category = summary['by_category'] + + # Check critical categories + critical_issues = [] + + for cat_num, cat_info in CATEGORIES.items(): + if not cat_info['critical']: + continue + + if cat_num not in by_category: + continue + + cat_stats = by_category[cat_num] + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + if pass_rate < 70: + critical_issues.append({ + 'category': f"{cat_num}.* {cat_info['name']}", + 'pass_rate': pass_rate, + 'description': cat_info['description'] + }) + + if critical_issues: + print(f"{Colors.RED}⚠️ Critical Issues Found:{Colors.END}\n") + for issue in critical_issues: + print(f" β€’ {issue['category']}: {issue['pass_rate']:.1f}% pass rate") + print(f" {issue['description']} - This requires attention\n") + else: + print(f"{Colors.GREEN}βœ… All critical test categories are performing well!{Colors.END}\n") + + # UTF-8 check + if '6' in by_category: + cat_stats = by_category['6'] + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + if pass_rate < 50: + print(f"{Colors.YELLOW}ℹ️ UTF-8 Validation (6.*): {pass_rate:.1f}% pass rate{Colors.END}") + print(f" This is acceptable for embedded clients in trusted environments.") + print(f" Consider adding UTF-8 validation if operating in untrusted networks.\n") + + print(f"{Colors.CYAN}πŸ’‘ Next Steps:{Colors.END}\n") + print(f" 1. Review failed tests in detail (see HTML report)") + print(f" 2. Focus on critical categories (1-5, 7, 10)") + print(f" 3. UTF-8 failures (category 6) are acceptable for embedded clients") + print(f" 4. Compare your results with reference implementations\n") + +def generate_html_summary(summary, results, output_file): + """Generate an HTML summary file""" + html = f""" + + + ESP WebSocket Client - Autobahn Test Summary + + + +
+

πŸ”¬ ESP WebSocket Client - Autobahn Test Suite Results

+

Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

+ +

πŸ“Š Overall Results

+
+
+

{summary['total']}

+

Total Tests

+
+
+

{summary['by_behavior'].get('OK', 0)}

+

Passed ({summary['by_behavior'].get('OK', 0)/summary['total']*100:.1f}%)

+
+
+

{summary['by_behavior'].get('FAILED', 0)}

+

Failed ({summary['by_behavior'].get('FAILED', 0)/summary['total']*100:.1f}%)

+
+
+ +
+
+ {summary['by_behavior'].get('OK', 0)/summary['total']*100:.1f}% Pass Rate +
+
+ +
+ + Grade: {calculate_grade(summary['by_behavior'].get('OK', 0)/summary['total']*100)[0]} + +
+ +

πŸ“‹ Results by Category

+ + + + + + + + + + + + + +""" + + # Add category rows + sorted_categories = sorted(summary['by_category'].keys(), key=lambda x: int(x) if x.isdigit() else 999) + for cat_num in sorted_categories: + cat_stats = summary['by_category'][cat_num] + cat_info = CATEGORIES.get(cat_num, {'name': f'Category {cat_num}', 'critical': True, 'description': 'Unknown'}) + + total = sum(cat_stats.values()) + passed = cat_stats.get('OK', 0) + failed = cat_stats.get('FAILED', 0) + pass_rate = (passed / total * 100) if total > 0 else 0 + + html += f""" + + + + + + + + + +""" + + html += """ + +
CategoryDescriptionTotalPassedFailedPass RateCritical
{cat_num}.*{cat_info['name']} - {cat_info['description']}{total}{passed}{failed} + {pass_rate:.1f}% + {'βœ… Yes' if cat_info['critical'] else 'βšͺ No'}
+ +

❌ Failed Tests

+ + + + + + + + + +""" + + failed_tests = [r for r in results if r['behavior'] == 'FAILED'] + for test in failed_tests[:50]: # Limit to 50 in HTML + html += f""" + + + + + +""" + + if len(failed_tests) > 50: + html += f""" + + + +""" + + html += """ + +
Test IDDescriptionResult
{test['id']}{test['description']}{test['result'][:100]}
+ ... and {len(failed_tests) - 50} more failed tests (see individual reports) +
+ +

πŸ’‘ Recommendations

+
+

For Embedded WebSocket Clients:

+
    +
  • βœ… Focus on passing critical categories: 1.* (Framing), 2.* (Ping/Pong), 5.* (Fragmentation), 7.* (Close)
    • +
  • ⚠️ UTF-8 validation failures (6.*) are acceptable in trusted environments
    • +
  • 🎯 Target >70% overall pass rate for production use
    • +
  • πŸ” Review individual test reports for specific implementation issues
    • +
+
+ +
+

+ View Detailed Reports: + Open Full Autobahn Report +

+
+
+ + +""" + + with open(output_file, 'w') as f: + f.write(html) + + print(f"{Colors.GREEN}βœ… HTML summary generated: {output_file}{Colors.END}\n") + +def main(): + """Main entry point""" + script_dir = Path(__file__).parent + reports_dir = script_dir.parent / 'reports' + + if not reports_dir.exists(): + print(f"{Colors.RED}Error: Reports directory not found: {reports_dir}{Colors.END}") + return 1 + + print(f"{Colors.CYAN}πŸ“– Parsing test results from: {reports_dir}{Colors.END}") + results = parse_test_results(str(reports_dir)) + + if not results: + print(f"{Colors.RED}Error: No test results found{Colors.END}") + return 1 + + print(f"{Colors.GREEN}βœ… Parsed {len(results)} test results{Colors.END}") + + summary = generate_summary(results) + + # Print console summary + print_summary_table(summary) + print_category_breakdown(summary) + print_failed_tests(results) + print_recommendations(summary) + + # Generate HTML summary + html_output = reports_dir / 'summary.html' + generate_html_summary(summary, results, str(html_output)) + + print(f"{Colors.CYAN}🌐 Open the summary in your browser:{Colors.END}") + print(f" file://{html_output.absolute()}\n") + + return 0 + +if __name__ == '__main__': + exit(main()) + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/scripts/monitor_serial.py b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/monitor_serial.py new file mode 100755 index 0000000000..a2034f5a53 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/monitor_serial.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Serial monitor script for Autobahn test suite. +Monitors ESP32 serial output and detects test completion. +""" + +import serial +import sys +import time +import re +import argparse + + +def main(): + parser = argparse.ArgumentParser(description='Monitor ESP32 serial output for Autobahn test completion') + parser.add_argument('--port', '-p', + default='/dev/ttyUSB0', + help='Serial port (default: /dev/ttyUSB0)') + parser.add_argument('--baud', '-b', + type=int, + default=115200, + help='Baud rate (default: 115200)') + parser.add_argument('--timeout', '-t', + type=int, + default=2400, + help='Timeout in seconds (default: 2400 = 40 minutes)') + parser.add_argument('--completion-pattern', '-c', + default=r'All tests completed\.', + help='Regex pattern to detect completion (default: "All tests completed.")') + parser.add_argument('--uri', '-u', + default=None, + help='Server URI to send via serial (stdin). If provided, will send this URI after opening port.') + + args = parser.parse_args() + + port = args.port + timeout_seconds = args.timeout + completion_pattern = re.compile(args.completion_pattern) + + print(f"Opening serial port: {port} at {args.baud} baud") + try: + ser = serial.Serial(port, args.baud, timeout=1) + print("Serial port opened successfully") + + # If URI is provided, send it via serial (stdin) + if args.uri: + print(f"Sending server URI: {args.uri}") + # Wait a bit for ESP32 to be ready + time.sleep(2) + # Send URI followed by newline + ser.write(f"{args.uri}\n".encode('utf-8')) + ser.flush() + print("URI sent successfully") + + buffer = "" + start_time = time.time() + + while True: + elapsed = time.time() - start_time + if elapsed > timeout_seconds: + print(f"\n⚠ Timeout after {timeout_seconds}s - tests may still be running") + sys.exit(1) + + if ser.in_waiting: + data = ser.read(ser.in_waiting).decode('utf-8', errors='ignore') + buffer += data + sys.stdout.write(data) + sys.stdout.flush() + + # Check for completion message + if completion_pattern.search(buffer): + print("\nβœ“ Test suite completed successfully!") + time.sleep(5) # Wait a bit more for any final output + sys.exit(0) + + time.sleep(0.1) + + ser.close() + except serial.SerialException as e: + print(f"Error opening serial port: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + print("\nInterrupted by user") + sys.exit(1) + + +if __name__ == '__main__': + main() + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/scripts/quick_test.sh b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/quick_test.sh new file mode 100755 index 0000000000..06df790689 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/quick_test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Quick Test Script - Runs a minimal set of tests for rapid validation +# This is useful for development/debugging without waiting for all 300+ tests + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +cd "$SCRIPT_DIR" + +echo "==========================================" +echo "Quick Test Mode - Core Tests Only" +echo "==========================================" +echo + +# Stop any existing server +docker-compose down 2>/dev/null || true + +# Copy quick config +cp config/fuzzingserver-quick.json config/fuzzingserver.json.backup 2>/dev/null || true +cp config/fuzzingserver-quick.json config/fuzzingserver.json + +echo "Starting fuzzing server with quick test config..." +echo " Tests: Core protocol tests (~150 cases)" +echo " Excludes: UTF-8 validation, performance, compression" +echo + +docker-compose up -d + +echo +echo "βœ“ Quick test server started" +echo +echo "Run your ESP32 testee client now." +echo "Estimated time: 5-10 minutes" +echo +echo "View results at: http://localhost:8080" +echo +echo "To restore full test config:" +echo " ./scripts/restore_full_tests.sh" + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/scripts/restore_full_tests.sh b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/restore_full_tests.sh new file mode 100755 index 0000000000..5b09c152c8 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/scripts/restore_full_tests.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Restore Full Test Configuration + +set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +cd "$SCRIPT_DIR" + +echo "Restoring full test configuration..." + +# Stop server +docker-compose down + +# Restore original config +if [ -f config/fuzzingserver.json.backup ]; then + mv config/fuzzingserver.json.backup config/fuzzingserver.json + echo "βœ“ Original config restored" +else + # Create default full config + cat > config/fuzzingserver.json << 'EOF' +{ + "url": "ws://0.0.0.0:9001", + "options": { + "failByDrop": false + }, + "outdir": "/reports", + "webport": 8080, + "cases": ["*"], + "exclude-cases": [ + "9.*", + "12.*", + "13.*" + ], + "exclude-agent-cases": {} +} +EOF + echo "βœ“ Default full config created" +fi + +echo +echo "Full test configuration restored (~300 tests)" +echo "Start server with: docker-compose up -d" + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/testee/CMakeLists.txt b/components/esp_websocket_client/examples/autobahn-testsuite/testee/CMakeLists.txt new file mode 100644 index 0000000000..3cd16c76d5 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/testee/CMakeLists.txt @@ -0,0 +1,9 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +# Component dependencies are declared in main/idf_component.yml + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(autobahn_testee) + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/CMakeLists.txt b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/CMakeLists.txt new file mode 100644 index 0000000000..6d6fca0439 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/CMakeLists.txt @@ -0,0 +1,4 @@ +idf_component_register(SRCS "autobahn_testee.c" + INCLUDE_DIRS "." + REQUIRES esp_websocket_client protocol_examples_common nvs_flash esp_wifi esp_event) + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/Kconfig.projbuild b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/Kconfig.projbuild new file mode 100644 index 0000000000..f2e7901c85 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/Kconfig.projbuild @@ -0,0 +1,37 @@ +menu "Autobahn Testsuite Configuration" + + choice WEBSOCKET_URI_SOURCE + prompt "Autobahn Server URI Source" + default WEBSOCKET_URI_FROM_STRING + help + Choose how the Autobahn server URI is provided: + - From string: Use CONFIG_AUTOBAHN_SERVER_URI (compile-time) + - From stdin: Read URI from serial console at runtime + + config WEBSOCKET_URI_FROM_STRING + bool "From string (CONFIG_AUTOBAHN_SERVER_URI)" + help + Use the URI defined in CONFIG_AUTOBAHN_SERVER_URI. + This is set at compile time. + + config WEBSOCKET_URI_FROM_STDIN + bool "From stdin (serial console)" + help + Read the URI from stdin (serial console) at runtime. + Useful for CI/CD where the server IP is only known at runtime. + The application will wait for a line containing the URI. + + endchoice + + config AUTOBAHN_SERVER_URI + string "Autobahn Fuzzing Server URI" + default "ws://192.168.1.100:9001" + depends on WEBSOCKET_URI_FROM_STRING + help + URI of the Autobahn fuzzing server. + Replace with your Docker host IP address. + Example: ws://192.168.1.100:9001 + Only used when WEBSOCKET_URI_FROM_STRING is selected. + +endmenu + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/autobahn_testee.c b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/autobahn_testee.c new file mode 100644 index 0000000000..b29fee9e39 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/autobahn_testee.c @@ -0,0 +1,468 @@ +#include +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include "esp_log.h" +#include "esp_timer.h" +#include "esp_system.h" +#include "esp_heap_caps.h" +#include "esp_event.h" +#include "esp_netif.h" +#include "nvs_flash.h" +#include "esp_wifi.h" +#include "protocol_examples_common.h" +#include "esp_websocket_client.h" +#include "esp_transport_ws.h" + +#define TAG "autobahn" +#if CONFIG_WEBSOCKET_URI_FROM_STDIN +// URI will be read from stdin at runtime +static char g_autobahn_server_uri[256] = {0}; +#define AUTOBAHN_SERVER_URI g_autobahn_server_uri +#else +#define AUTOBAHN_SERVER_URI CONFIG_AUTOBAHN_SERVER_URI +#endif +#define BUFFER_SIZE 16384 // Reduced from 32768 to free memory for accumulator buffer +#define START_CASE 1 +#define END_CASE 16 +// Configure test range here: +// Category 1 (Framing): Tests 1-16 +// Category 2 (Ping/Pong): Tests 17-27 +// Category 3 (Reserved Bits): Tests 28-34 +// Category 4 (Opcodes): Tests 35-44 +// Category 5 (Fragmentation): Tests 45-64 +// Category 6 (UTF-8): Tests 65-209 +// Category 7 (Close Handshake): Tests 210-246 +// All tests: Tests 1-300 +static SemaphoreHandle_t test_done_sem = NULL; +static bool test_running = false; + +#define MAX_FRAGMENTED_PAYLOAD 65537 // Maximum payload size for fragmented frames (case 1.1.6=65535, 1.1.7=65536) + +typedef struct { + uint8_t *buffer; + size_t capacity; + size_t expected_len; + size_t received; + uint8_t opcode; + bool active; +} ws_accumulator_t; + +static ws_accumulator_t s_accumulator = {0}; +static uint8_t *s_accum_buffer = NULL; // Pre-allocated buffer for fragmented frames + +static void ws_accumulator_reset(void) +{ + // Reset state but keep buffer allocated for reuse + s_accumulator.expected_len = 0; + s_accumulator.received = 0; + s_accumulator.opcode = 0; + s_accumulator.active = false; +} + +static void ws_accumulator_cleanup(void) +{ + ws_accumulator_reset(); + if (s_accum_buffer) { + free(s_accum_buffer); + s_accum_buffer = NULL; + ESP_LOGD(TAG, "Freed accumulator buffer"); + } +} + +static esp_err_t ws_accumulator_prepare(size_t total_len, uint8_t opcode) +{ + if (total_len == 0) { + return ESP_OK; + } + + if (total_len > MAX_FRAGMENTED_PAYLOAD) { + ESP_LOGE(TAG, "Payload too large (%zu > %d)", total_len, MAX_FRAGMENTED_PAYLOAD); + return ESP_ERR_INVALID_SIZE; + } + + // Allocate buffer on-demand when first fragmented frame is detected + // This avoids allocating 64KB upfront which can cause memory exhaustion + if (!s_accum_buffer) { + size_t free_heap = esp_get_free_heap_size(); + size_t largest_free = heap_caps_get_largest_free_block(MALLOC_CAP_DEFAULT); + ESP_LOGD(TAG, "Attempting accumulator alloc: need=%zu, free=%zu, largest_block=%zu", + MAX_FRAGMENTED_PAYLOAD, free_heap, largest_free); + + s_accum_buffer = (uint8_t *)malloc(MAX_FRAGMENTED_PAYLOAD); + if (!s_accum_buffer) { + ESP_LOGE(TAG, "Accumulator alloc failed (%zu bytes) - Free heap: %zu, largest block: %zu", + total_len, free_heap, largest_free); + ESP_LOGE(TAG, "ESP32-S2 may not have enough RAM. Consider reducing BUFFER_SIZE or using SPIRAM"); + return ESP_ERR_NO_MEM; + } + ESP_LOGD(TAG, "Allocated accumulator buffer: %d bytes (Free heap: %lu)", + MAX_FRAGMENTED_PAYLOAD, esp_get_free_heap_size()); + } + + s_accumulator.buffer = s_accum_buffer; + s_accumulator.capacity = MAX_FRAGMENTED_PAYLOAD; + s_accumulator.expected_len = total_len; + s_accumulator.received = 0; + s_accumulator.opcode = opcode; + s_accumulator.active = true; + return ESP_OK; +} + +/* ------------------------------------------------------------ + * Low‑latency echo handler + * ------------------------------------------------------------ */ +static void websocket_event_handler(void *handler_args, + esp_event_base_t base, + int32_t event_id, + void *event_data) +{ + esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; + esp_websocket_client_handle_t client = (esp_websocket_client_handle_t)handler_args; + + switch (event_id) { + case WEBSOCKET_EVENT_CONNECTED: + ESP_LOGI(TAG, "Connected"); + test_running = true; + break; + + case WEBSOCKET_EVENT_DISCONNECTED: + ESP_LOGI(TAG, "Disconnected"); + test_running = false; + ws_accumulator_reset(); // Reset state but keep buffer for next test + if (test_done_sem) xSemaphoreGive(test_done_sem); + break; + + case WEBSOCKET_EVENT_DATA: { + ESP_LOGI(TAG, "WEBSOCKET_EVENT_DATA: opcode=0x%02X len=%d fin=%d payload_len=%d offset=%d", + data->op_code, data->data_len, data->fin, data->payload_len, data->payload_offset); + /* ---- skip control frames ---- */ + if (data->op_code >= 0x08) { + if (data->op_code == 0x09) + ESP_LOGD(TAG, "PING -> PONG auto-sent"); + break; + } + + /* ---- Determine opcode to echo ---- */ + uint8_t send_opcode = 0; + if (data->op_code == 0x1) { + send_opcode = WS_TRANSPORT_OPCODES_TEXT; + } else if (data->op_code == 0x2) { + send_opcode = WS_TRANSPORT_OPCODES_BINARY; + } else if (data->op_code == 0x0) { + send_opcode = WS_TRANSPORT_OPCODES_CONT; + } else { + ESP_LOGW(TAG, "Unsupported opcode 0x%02X - skip", data->op_code); + break; + } + + /* Note: send_with_opcode always sets FIN bit, which is correct for these + * simple test cases (all have FIN=1). For fragmented messages, we'd need + * send_with_exact_opcode, but it's not public. */ + + const uint8_t *payload = (const uint8_t *)data->data_ptr; + size_t len = data->data_len; + + // Check if this is a fragmented message (either WebSocket fragmentation or TCP-level fragmentation) + // The WebSocket layer reads large frames in chunks and dispatches multiple events: + // - payload_len = total frame size (set on all chunks) + // - payload_offset = current offset (0, buffer_size, 2*buffer_size, ...) + // - data_len = current chunk size + // - fin = 1 only on the last chunk + // So fragmentation is detected if: payload_len > data_len OR payload_offset > 0 + size_t total_len = data->payload_len ? data->payload_len : data->data_len; + bool fragmented = (data->payload_len > 0 && data->payload_len > data->data_len) || + (data->payload_offset > 0); + + ESP_LOGD(TAG, "Fragmentation check: offset=%d payload_len=%d data_len=%d total_len=%zu fragmented=%d", + data->payload_offset, data->payload_len, data->data_len, total_len, fragmented); + + if (fragmented && total_len > 0) { + if (data->payload_offset == 0 || !s_accumulator.active) { + if (ws_accumulator_prepare(total_len, send_opcode) != ESP_OK) { + ESP_LOGE(TAG, "Cannot allocate buffer for fragmented frame len=%zu", total_len); + break; + } + } else if (total_len != s_accumulator.expected_len) { + ESP_LOGW(TAG, "Payload len changed mid-message (%zu -> %zu) - reset accumulator", + s_accumulator.expected_len, total_len); + ws_accumulator_reset(); + if (ws_accumulator_prepare(total_len, send_opcode) != ESP_OK) { + break; + } + } + + if (!s_accumulator.active) { + ESP_LOGE(TAG, "Accumulator inactive while processing fragments"); + break; + } + + size_t offset = data->payload_offset; + if (offset + data->data_len > s_accumulator.capacity) { + ESP_LOGE(TAG, "Accumulator overflow: off=%zu chunk=%d cap=%zu", + offset, data->data_len, s_accumulator.capacity); + ws_accumulator_reset(); + break; + } + memcpy(s_accumulator.buffer + offset, data->data_ptr, data->data_len); + s_accumulator.received = offset + data->data_len; + + if (s_accumulator.received < s_accumulator.expected_len) { + // wait for more fragments + break; + } + + // Completed full message + payload = s_accumulator.buffer; + len = s_accumulator.expected_len; + send_opcode = s_accumulator.opcode; + s_accumulator.active = false; + } + + int sent = -1; + int attempt = 0; + const TickType_t backoff[] = {1, 1, 1, 2, 4, 8}; // Shorter backoff for faster retry + int64_t start = esp_timer_get_time(); + + /* Send echo immediately - use timeout scaled by frame size for large frames */ + /* For large messages (>16KB), the send function fragments into multiple chunks */ + /* Each chunk needs sufficient timeout, so scale timeout per chunk, not total message */ + while (sent < 0 && esp_websocket_client_is_connected(client)) { + /* For zero-length payload, pass NULL pointer (API handles this correctly) */ + /* Calculate timeout per chunk: large messages are fragmented, each chunk needs time */ + TickType_t send_timeout = pdMS_TO_TICKS(5); // Default 5ms for small frames + if (len > 1024) { + // For large messages, use a per-chunk timeout that accounts for network delays + // Since messages are fragmented into ~16KB chunks, each chunk needs sufficient time + // Use a fixed generous timeout per chunk for large messages (500ms per chunk) + // For 65535 bytes = 4 chunks, total time could be up to 2 seconds + send_timeout = pdMS_TO_TICKS(500); // 500ms per chunk for large messages + } else { + // Small messages: scale timeout based on size + send_timeout = pdMS_TO_TICKS((len / 256) + 10); + if (send_timeout > pdMS_TO_TICKS(100)) { + send_timeout = pdMS_TO_TICKS(100); + } + } + + ESP_LOGD(TAG, "Sending echo: opcode=0x%02X len=%zu timeout=%lums", + send_opcode, len, (unsigned long)(send_timeout * portTICK_PERIOD_MS)); + + sent = esp_websocket_client_send_with_opcode( + client, send_opcode, + (len > 0) ? payload : NULL, len, + send_timeout); + + if (sent >= 0) { + ESP_LOGD(TAG, "Echo sent successfully: %d bytes", sent); + break; + } + ESP_LOGW(TAG, + "echo send retry: opcode=0x%02X len=%zu fin=%d attempt=%d sent=%d", + send_opcode, len, data->fin, attempt + 1, sent); + if (attempt < (int)(sizeof(backoff)/sizeof(backoff[0]))) + vTaskDelay(backoff[attempt++]); + else + vTaskDelay(32); + } + + int64_t dt = esp_timer_get_time() - start; + if (sent >= 0) { + ESP_LOGI(TAG, "Echo success: opcode=0x%02X len=%d fin=%d in %lldus", + data->op_code, sent, data->fin, (long long)dt); + } else { + ESP_LOGE(TAG, "Echo failed: opcode=0x%02X len=%d fin=%d", + data->op_code, (int)len, data->fin); + } + break; + } + + case WEBSOCKET_EVENT_ERROR: + case WEBSOCKET_EVENT_FINISH: + test_running = false; + if (test_done_sem) xSemaphoreGive(test_done_sem); + break; + default: + break; + } +} + +/* ------------------------------------------------------------ */ +static esp_err_t run_test_case(int case_num) +{ + char uri[512]; // Increased to accommodate full URI + path + int ret = snprintf(uri, sizeof(uri), + "%s/runCase?case=%d&agent=esp_websocket_client", + AUTOBAHN_SERVER_URI, case_num); + if (ret < 0 || ret >= (int)sizeof(uri)) { + ESP_LOGE(TAG, "URI too long: %s/runCase?case=%d&agent=esp_websocket_client", AUTOBAHN_SERVER_URI, case_num); + return ESP_ERR_INVALID_ARG; + } + ESP_LOGI(TAG, "Running case %d: %s", case_num, uri); + + esp_websocket_client_config_t cfg = { + .uri = uri, + .buffer_size = BUFFER_SIZE, + .network_timeout_ms = 10000, // 10s for connection (default), 200ms was too short + .reconnect_timeout_ms = 500, + .task_prio = 10, // High prio β†’ low latency + .task_stack = 8144, + }; + + // If accumulator buffer is not allocated yet, try to allocate it now + // (before client init to avoid fragmentation) + if (!s_accum_buffer) { + ESP_LOGD(TAG, "Attempting to allocate accumulator buffer before client init (Free heap: %lu)", + esp_get_free_heap_size()); + s_accum_buffer = (uint8_t *)malloc(MAX_FRAGMENTED_PAYLOAD); + if (s_accum_buffer) { + ESP_LOGD(TAG, "Successfully allocated accumulator buffer: %d bytes", MAX_FRAGMENTED_PAYLOAD); + } + } + + esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg); + if (!client) return ESP_FAIL; + + esp_websocket_register_events(client, WEBSOCKET_EVENT_ANY, + websocket_event_handler, (void*)client); + + test_done_sem = xSemaphoreCreateBinary(); + + esp_websocket_client_start(client); + + /* Wait up to 60β€―s so server can close properly */ + xSemaphoreTake(test_done_sem, pdMS_TO_TICKS(60000)); + + if (esp_websocket_client_is_connected(client)) + esp_websocket_client_stop(client); + + esp_websocket_client_destroy(client); + vSemaphoreDelete(test_done_sem); + test_done_sem = NULL; + ESP_LOGI(TAG, "Free heap: %lu", esp_get_free_heap_size()); + return ESP_OK; +} + +/* ------------------------------------------------------------ */ +static void update_reports(void) +{ + char uri[512]; // Increased to accommodate full URI + path + int ret = snprintf(uri, sizeof(uri), + "%s/updateReports?agent=esp_websocket_client", + AUTOBAHN_SERVER_URI); + if (ret < 0 || ret >= (int)sizeof(uri)) { + ESP_LOGE(TAG, "URI too long: %s/updateReports?agent=esp_websocket_client", AUTOBAHN_SERVER_URI); + return; + } + esp_websocket_client_config_t cfg = { .uri = uri }; + esp_websocket_client_handle_t client = esp_websocket_client_init(&cfg); + esp_websocket_client_start(client); + vTaskDelay(pdMS_TO_TICKS(3000)); + esp_websocket_client_stop(client); + esp_websocket_client_destroy(client); + ESP_LOGI(TAG, "Reports updated"); +} + +/* ------------------------------------------------------------ */ +static void websocket_app_start(void) +{ + ESP_LOGI(TAG, "===================================="); + ESP_LOGI(TAG, " Autobahn WebSocket Testsuite Client"); + ESP_LOGI(TAG, "===================================="); + ESP_LOGI(TAG, "Server: %s", AUTOBAHN_SERVER_URI); + + // Accumulator buffer should already be allocated in app_main() before any clients + // If not, it will be allocated on-demand when first fragmented frame is detected + if (s_accum_buffer) { + ESP_LOGI(TAG, "Accumulator buffer ready: %d bytes", MAX_FRAGMENTED_PAYLOAD); + } else { + ESP_LOGW(TAG, "Accumulator buffer not pre-allocated, will allocate on-demand (max %d bytes)", MAX_FRAGMENTED_PAYLOAD); + } + + for (int i = START_CASE; i <= END_CASE; i++) { + ESP_LOGI(TAG, "========== Case %d/%d ==========", i, END_CASE); + run_test_case(i); + vTaskDelay(pdMS_TO_TICKS(500)); + } + update_reports(); + + // Free accumulator buffer after all tests + ws_accumulator_cleanup(); + ESP_LOGI(TAG, "All tests completed."); +} + +#if CONFIG_WEBSOCKET_URI_FROM_STDIN +/* ------------------------------------------------------------ + * Read URI from stdin (similar to websocket_example.c) + * ------------------------------------------------------------ */ +static void get_string(char *line, size_t size) +{ + int count = 0; + while (count < size - 1) { + int c = fgetc(stdin); + if (c == '\n' || c == '\r') { + line[count] = '\0'; + break; + } else if (c > 0 && c < 127) { + line[count] = c; + ++count; + } + vTaskDelay(10 / portTICK_PERIOD_MS); + } + if (count == size - 1) { + line[count] = '\0'; + } +} +#endif /* CONFIG_WEBSOCKET_URI_FROM_STDIN */ + +/* ------------------------------------------------------------ */ +void app_main(void) +{ + ESP_LOGI(TAG, "Startup, IDF %s", esp_get_idf_version()); + ESP_ERROR_CHECK(nvs_flash_init()); + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + + // Allocate accumulator buffer early, before any WebSocket clients are created + // This ensures we have enough contiguous memory before heap gets fragmented + // ESP32-S2 has limited RAM (~320KB total), so we need to allocate this early + ESP_LOGI(TAG, "Allocating accumulator buffer early (Free heap: %lu)", esp_get_free_heap_size()); + s_accum_buffer = (uint8_t *)malloc(MAX_FRAGMENTED_PAYLOAD); + if (!s_accum_buffer) { + ESP_LOGE(TAG, "Failed to allocate accumulator buffer (%d bytes) - Free heap: %lu", + MAX_FRAGMENTED_PAYLOAD, esp_get_free_heap_size()); + ESP_LOGE(TAG, "ESP32-S2 may not have enough RAM for 64KB buffer. Consider:"); + ESP_LOGE(TAG, " 1. Reducing BUFFER_SIZE further (currently %d)", BUFFER_SIZE); + ESP_LOGE(TAG, " 2. Using SPIRAM if available"); + ESP_LOGE(TAG, " 3. Skipping large payload tests (case 1.1.6)"); + // Continue anyway - will try on-demand allocation later + } else { + ESP_LOGI(TAG, "Successfully allocated accumulator buffer: %d bytes (Free heap: %lu)", + MAX_FRAGMENTED_PAYLOAD, esp_get_free_heap_size()); + } + + ESP_ERROR_CHECK(example_connect()); + /* disable power‑save for low latency */ + esp_wifi_set_ps(WIFI_PS_NONE); + +#if CONFIG_WEBSOCKET_URI_FROM_STDIN + // Read server URI from stdin + ESP_LOGI(TAG, "Waiting for Autobahn server URI from stdin..."); + ESP_LOGI(TAG, "Please send URI in format: ws://:9001"); + get_string(g_autobahn_server_uri, sizeof(g_autobahn_server_uri)); + ESP_LOGI(TAG, "Received server URI: %s", g_autobahn_server_uri); + + if (strlen(g_autobahn_server_uri) == 0) { + ESP_LOGE(TAG, "No URI received from stdin, using default"); + #ifdef CONFIG_AUTOBAHN_SERVER_URI + strncpy(g_autobahn_server_uri, CONFIG_AUTOBAHN_SERVER_URI, sizeof(g_autobahn_server_uri) - 1); + #else + strncpy(g_autobahn_server_uri, "ws://192.168.1.100:9001", sizeof(g_autobahn_server_uri) - 1); + #endif + } +#endif + + websocket_app_start(); +} diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/idf_component.yml b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/idf_component.yml new file mode 100644 index 0000000000..a2e0539fcf --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/testee/main/idf_component.yml @@ -0,0 +1,11 @@ +dependencies: + ## Required IDF version + idf: ">=5.0" + # WebSocket client component from parent directory + espressif/esp_websocket_client: + version: "^1.0.0" + override_path: "../../../../" + # WiFi connection helper from ESP-IDF examples + protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + diff --git a/components/esp_websocket_client/examples/autobahn-testsuite/testee/sdkconfig.ci.plain_tcp b/components/esp_websocket_client/examples/autobahn-testsuite/testee/sdkconfig.ci.plain_tcp new file mode 100644 index 0000000000..6efc580133 --- /dev/null +++ b/components/esp_websocket_client/examples/autobahn-testsuite/testee/sdkconfig.ci.plain_tcp @@ -0,0 +1,15 @@ +CONFIG_IDF_TARGET="esp32" +CONFIG_IDF_TARGET_LINUX=n +CONFIG_WEBSOCKET_URI_FROM_STDIN=y +CONFIG_WEBSOCKET_URI_FROM_STRING=n +CONFIG_EXAMPLE_CONNECT_ETHERNET=y +CONFIG_EXAMPLE_CONNECT_WIFI=n +CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y +CONFIG_EXAMPLE_ETH_PHY_IP101=y +CONFIG_EXAMPLE_ETH_MDC_GPIO=23 +CONFIG_EXAMPLE_ETH_MDIO_GPIO=18 +CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5 +CONFIG_EXAMPLE_ETH_PHY_ADDR=1 +CONFIG_EXAMPLE_CONNECT_IPV6=y +CONFIG_WS_OVER_TLS_MUTUAL_AUTH=n +CONFIG_WS_OVER_TLS_SERVER_AUTH=n