feat: HTTPProvider with AutoTLS

.github/workflows/gateway-conformance.yml

@@ -60,11 +60,17 @@ jobs:
         run: make build
         working-directory: kubo-gateway
 
-      # 3. Init the kubo-gateway
+      # 3. Init the kubo-gateway. Every Addresses.* port is set explicitly
+      # and chosen unique across all three conformance jobs (leading 1xxxx
+      # for this one, 2xxxx and 3xxxx for the libp2p and cleartext jobs)
+      # so each job's footprint is unambiguous.
       - name: Init kubo-gateway
         run: |
           ./ipfs init -e
           ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/14001"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/18080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/15001"
         working-directory: kubo-gateway/cmd/ipfs
 
       # 4. Populate the Kubo gateway with the gateway-conformance fixtures
@@ -95,8 +101,8 @@ jobs:
       - name: Run gateway-conformance tests
         uses: ipfs/gateway-conformance/.github/actions/test@v0.13
         with:
-          gateway-url: http://127.0.0.1:8080
-          subdomain-url: http://localhost:8080
+          gateway-url: http://127.0.0.1:18080
+          subdomain-url: http://localhost:18080
           args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
           json: output.json
           xml: output.xml
@@ -146,14 +152,19 @@ jobs:
         run: make build
         working-directory: kubo-gateway
 
-      # 3. Init the kubo-gateway
+      # 3. Init the kubo-gateway. The trustless subset this job runs does
+      # not exercise subdomain routing, so Gateway.PublicGateways is
+      # unnecessary here. Ports are in the 2xxxx range so each conformance
+      # job's footprint is unambiguous (1xxxx for the full job, 3xxxx for
+      # the cleartext job).
       - name: Init kubo-gateway
         run: |
           ./ipfs init --profile=test
-          ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
-          ./ipfs config --json Experimental.GatewayOverLibp2p true
-          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
-          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
+          ./ipfs config --json HTTPProvider.Enabled true
+          ./ipfs config --json HTTPProvider.Libp2p true
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/24001"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/28080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/25001"
         working-directory: kubo-gateway/cmd/ipfs
 
       # 4. Populate the Kubo gateway with the gateway-conformance fixtures
@@ -176,8 +187,9 @@ jobs:
         run: |
           ./ipfs init --profile=test -e
           ./ipfs config --json Experimental.Libp2pStreamMounting true
-          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081"
-          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002"
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/24002"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/28081"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/25002"
         working-directory: kubo-gateway/cmd/ipfs
 
       # 7. Start the kubo http-p2p-proxy
@@ -192,16 +204,16 @@ jobs:
       # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
       - name: Start http-over-libp2p forwarding proxy
         run: |
-          gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="<id>")
-          ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1)
-          ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8092 /p2p/$gatewayNodeId
+          gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/25001 id -f="<id>")
+          ./ipfs --api=/ip4/127.0.0.1/tcp/25002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/25001 swarm addrs local --id | head -n 1)
+          ./ipfs --api=/ip4/127.0.0.1/tcp/25002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/28092 /p2p/$gatewayNodeId
         working-directory: kubo-gateway/cmd/ipfs
 
       # 9. Run the gateway-conformance tests over libp2p
       - name: Run gateway-conformance tests over libp2p
         uses: ipfs/gateway-conformance/.github/actions/test@v0.13
         with:
-          gateway-url: http://127.0.0.1:8092
+          gateway-url: http://127.0.0.1:28092
           args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
           json: output.json
           xml: output.xml
@@ -224,3 +236,94 @@ jobs:
         with:
           name: gateway-conformance-libp2p.json
           path: output.json
+
+  # Testing the trustless gateway subset exposed by HTTPProvider over plain
+  # HTTP/2 (h2c) on the swarm port. HTTPProvider.Cleartext auto-appends a /ws
+  # listener to each /tcp listener in Addresses.Swarm; HTTPProvider then
+  # shares the same TCP port via the shared-TCP demuxer and serves /http
+  # there. No libp2p proxy is needed: the conformance client connects to the
+  # swarm port directly. Complements the libp2p job above so both
+  # HTTPProvider transports are exercised end-to-end.
+  gateway-conformance-http-provider-cleartext:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13
+        with:
+          output: fixtures
+
+      # 2. Build the kubo-gateway
+      - name: Checkout kubo-gateway
+        uses: actions/checkout@v6
+        with:
+          path: kubo-gateway
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'kubo-gateway/go.mod'
+          cache: true
+          cache-dependency-path: kubo-gateway/go.sum
+      - name: Build kubo-gateway
+        run: make build
+        working-directory: kubo-gateway
+
+      # 3. Init the kubo-gateway with HTTPProvider.Cleartext on the swarm
+      #    port. Libp2p is off; Cleartext is the only HTTPProvider transport
+      #    exercised by this job. The trustless subset run below does not
+      #    exercise subdomain routing, so Gateway.PublicGateways is
+      #    unnecessary here. Ports are in the 3xxxx range so each
+      #    conformance job's footprint is unambiguous (1xxxx for the full
+      #    job, 2xxxx for the libp2p job).
+      - name: Init kubo-gateway
+        run: |
+          ./ipfs init --profile=test
+          ./ipfs config --json HTTPProvider.Enabled true
+          ./ipfs config --json HTTPProvider.Libp2p false
+          ./ipfs config --json HTTPProvider.Cleartext true
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/34001"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/38080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/35001"
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 4. Populate the Kubo gateway with the gateway-conformance fixtures
+      - name: Import fixtures
+        run: |
+          # Import car files
+          find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
+
+      # 5. Start the kubo-gateway
+      - name: Start kubo-gateway
+        run: |
+          ( ./ipfs daemon & ) | sed '/Daemon is ready/q'
+          while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 6. Run the gateway-conformance tests over plain HTTP/2 (h2c) on the swarm port
+      - name: Run gateway-conformance tests over h2c
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.13
+        with:
+          gateway-url: http://127.0.0.1:34001
+          args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+
+      # 7. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gateway-conformance-http-provider-cleartext.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gateway-conformance-http-provider-cleartext.json
+          path: output.json

.github/workflows/gotest.yml

@@ -202,6 +202,28 @@ jobs:
                 fusermount3 -uz "$mp" 2>/dev/null || fusermount -uz "$mp" 2>/dev/null || true
               done
 
+  # AutoTLS end-to-end canary (in-process Pebble + p2p-forge).
+  # Isolated sub-module (test/autotls); heavy CoreDNS + Pebble deps stay
+  # out of the main go.mod.
+  autotls-tests:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
+    timeout-minutes: 8
+    env:
+      GOTRACEBACK: all
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+      - name: Build kubo binary
+        run: make build
+      - name: Run AutoTLS end-to-end canary
+        run: make test_autotls
+
   # Example tests (kubo-as-a-library)
   example-tests:
     if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'

.gitignore

@@ -34,6 +34,7 @@ docs/examples/go-ipfs-as-a-library/example-folder/Qm*
 /test/cli/cli-tests.json
 /test/fuse/fuse-unit-tests.json
 /test/fuse/fuse-cli-tests.json
+/test/autotls/autotls-tests.json
 
 # ignore build output from snapcraft
 /ipfs_*.snap

AGENTS.md

@@ -106,7 +106,7 @@ make -O test_go_lint # run linter (use this instead of golangci-lint directly)
 
 If you change dependencies in any `go.mod`, you must run `make mod_tidy`, and you must run it before committing, pushing, or opening a PR. The repo has three `go.mod` files (root, `docs/examples/kubo-as-a-library`, and `test/dependencies`) that have to stay on the same dependency versions. `make mod_tidy` runs `go mod tidy` in every one of them; a bare `go mod tidy` only touches the module you run it in, which lets the pins drift out of sync between modules (for example the root pointing at one boxo commit while `test/dependencies` points at another). Run it before building or testing too, since it also updates `go.sum`.
 
-If you modify any `.go` files outside of `test/`, you must run `make build` before running integration tests.
+If you modify any `.go` files outside of `test/`, you must run `make build` before running integration tests. Integration tests spawn a real `ipfs daemon` from `cmd/ipfs/ipfs` and `go test ./test/cli/...` does not rebuild that binary, so daemon-side changes (config schema, FX providers, transport options, default values) appear to be silently ignored when the binary is stale. Common symptoms: a new config flag has no effect, an expected listener never binds, an FX-injected handler is nil. If a CLI test behaves differently from a manual `cmd/ipfs/ipfs daemon` run, suspect a stale binary first.
 
 ## Testing
 

Rules.mk

@@ -139,6 +139,7 @@ help:
 	@echo '  test_unit               - Run unit tests with coverage (excludes test/cli)'
 	@echo '  test_cli                - Run CLI integration tests (requires built binary)'
 	@echo '  test_fuse               - Run FUSE tests (requires /dev/fuse and fusermount)'
+	@echo '  test_autotls            - Run AutoTLS end-to-end canary (in-process Pebble + p2p-forge)'
 	@echo '  test_go_fmt             - Check Go source formatting'
 	@echo '  test_go_build           - Build kubo for all platforms from .github/build-platforms.yml'
 	@echo '  test_go_lint            - Run golangci-lint'