Add fan-out proxy to collectors

.gitignore

@@ -1,9 +1,12 @@
 /target
 *.logjet
 logs/
+*.log
 **/logs/
 TODO*
 .*
 bsd*
 *.logjet
+/demo/*/*.state
+/demo/*/certs
 bridge.state

Cargo.toml

@@ -46,3 +46,4 @@ arrow-array = "58.0.0"
 opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs"] }
 parquet = { version = "58.0.0", default-features = false, features = ["arrow", "zstd"] }
 prost = "0.14"
+rand = "0.9.3"

demo/README.md

@@ -61,6 +61,8 @@ It also contains scenario demos under subdirectories:
   - same remote-drain topology, but with TLS and mutual TLS on the replay link
 - [`secure-pipeline`](./secure-pipeline)
   - HTTPS OTLP ingest into `ljd`, then HTTPS collector export on replay
+- [`proxy-to-vector`](./proxy-to-vector)
+  - appliance-side `ljd` replayed through `ljd bridge` into Vector stdout over OTLP/HTTP or OTLP/gRPC
 
 ## Enjoy It
 

demo/proxy-to-vector/README.md

@@ -0,0 +1,157 @@
+# Proxy To Vector Demo
+
+This demo sends OTLP logs through two `ljd` processes and into Vector stdout.
+
+Flow:
+
+```text
+OTLP logs -> appliance ljd -> bridge ljd -> Vector -> stdout
+```
+
+Directory:
+
+```text
+demo/proxy-to-vector
+```
+
+## Build First
+
+From the project root:
+
+```bash
+make demo
+```
+
+## Run Vector
+
+From this directory:
+
+```bash
+VECTOR_BIN=/usr/bin/vector ./run-vector.sh
+```
+
+`VECTOR_BIN` must be the full absolute path to the Vector binary.
+
+`run-vector.sh` also generates demo-only TLS files under `./certs` if they do
+not exist yet.
+
+## Run One Proxy Mode
+
+This demo contains four runner scripts:
+
+- `run-vector.sh`
+- `run-proxy-http.sh`
+- `run-proxy-grpc.sh`
+- `run-proxy-grpc-tls.sh`
+
+Run Vector in terminal 1, then choose one proxy mode in terminal 2.
+
+HTTP mode:
+
+```bash
+# terminal 1
+VECTOR_BIN=/usr/bin/vector ./run-vector.sh
+
+# terminal 2
+./run-proxy-http.sh
+```
+
+Plain gRPC mode:
+
+```bash
+# terminal 1
+VECTOR_BIN=/usr/bin/vector ./run-vector.sh
+
+# terminal 2
+./run-proxy-grpc.sh
+```
+
+gRPC over TLS mode:
+
+```bash
+# terminal 1
+VECTOR_BIN=/usr/bin/vector ./run-vector.sh
+
+# terminal 2
+./run-proxy-grpc-tls.sh
+```
+
+## Featurette: Run All Three
+
+You can run all three proxy modes at the same time against one Vector process.
+
+That works because:
+
+- Vector listens on:
+  - HTTP `127.0.0.1:4318`
+  - plain gRPC `127.0.0.1:4317`
+  - TLS gRPC `127.0.0.1:4417`
+- each demo mode uses different appliance and replay ports
+- each demo mode uses separate state and log files
+
+```bash
+# terminal 1
+VECTOR_BIN=/usr/bin/vector ./run-vector.sh
+
+# terminal 2
+./run-proxy-http.sh
+
+# terminal 3
+./run-proxy-grpc.sh
+
+# terminal 4
+./run-proxy-grpc-tls.sh
+```
+
+## Endpoints
+
+HTTP mode:
+
+- appliance-side `ljd` accepts OTLP/HTTP on `127.0.0.1:4319`
+- bridge-side `ljd` forwards OTLP/HTTP to Vector on `127.0.0.1:4318`
+
+Plain gRPC mode:
+
+- appliance-side `ljd` accepts OTLP/gRPC on `127.0.0.1:4329`
+- bridge-side `ljd` forwards OTLP/gRPC to Vector on `127.0.0.1:4317`
+
+gRPC TLS mode:
+
+- appliance-side `ljd` accepts OTLP/gRPC on `127.0.0.1:4339`
+- bridge-side `ljd` forwards OTLP/gRPC over TLS to Vector on `127.0.0.1:4417`
+- bridge-side `ljd` validates the Vector certificate with:
+  - `collector.ca-file: ./certs/ca.pem`
+  - `collector.server-name: vector.demo.logjet`
+
+## Certificates
+
+The TLS mode uses local demo-only certificates under:
+
+```text
+./certs
+```
+
+Files:
+
+- `ca.pem`
+- `ca.key`
+- `vector.pem`
+- `vector.key`
+
+These files are generated by `./certs/gen-certs.sh`.
+
+Do not use them anywhere real.
+
+## Local Files
+
+- `bridge-http.state`
+- `bridge-grpc.state`
+- `bridge-grpc-tls.state`
+- `appliance-http.log`
+- `bridge-http.log`
+- `appliance-grpc.log`
+- `bridge-grpc.log`
+- `appliance-grpc-tls.log`
+- `bridge-grpc-tls.log`
+
+These files are recreated inside this demo directory.

demo/proxy-to-vector/appliance-grpc-logjetd.conf

@@ -0,0 +1,8 @@
+output: buffer
+buffer.messages: 64
+buffer.keep: 8
+ingest.protocol: otlp-grpc
+ingest.listen: 127.0.0.1:4329
+replay.listen: 127.0.0.1:7012
+collector.url: grpc://127.0.0.1:4317
+collector.timeout-ms: 10000

demo/proxy-to-vector/appliance-grpc-tls-logjetd.conf

@@ -0,0 +1,8 @@
+output: buffer
+buffer.messages: 64
+buffer.keep: 8
+ingest.protocol: otlp-grpc
+ingest.listen: 127.0.0.1:4339
+replay.listen: 127.0.0.1:7022
+collector.url: grpc://127.0.0.1:4317
+collector.timeout-ms: 10000

demo/proxy-to-vector/appliance-http-logjetd.conf

@@ -0,0 +1,8 @@
+output: buffer
+buffer.messages: 64
+buffer.keep: 8
+ingest.protocol: otlp-http
+ingest.listen: 127.0.0.1:4319
+replay.listen: 127.0.0.1:7002
+collector.url: http://127.0.0.1:4318/v1/logs
+collector.timeout-ms: 10000

demo/proxy-to-vector/bridge-grpc-logjetd.conf

@@ -0,0 +1,13 @@
+output: buffer
+buffer.messages: 16
+buffer.keep: 0
+collector.url: grpc://127.0.0.1:4317
+collector.timeout-ms: 10000
+upstream.replay: 127.0.0.1:7012
+upstream.mode: keep
+upstream.state-file: ./bridge-grpc.state
+upstream.retry-ms: 1000
+upstream.connect-timeout-ms: 5000
+ingest.protocol: wire
+ingest.listen: 127.0.0.1:7011
+replay.listen: 127.0.0.1:7013

demo/proxy-to-vector/bridge-grpc-tls-logjetd.conf

@@ -0,0 +1,15 @@
+output: buffer
+buffer.messages: 16
+buffer.keep: 0
+collector.url: grpcs://127.0.0.1:4417
+collector.timeout-ms: 10000
+collector.ca-file: ./certs/ca.pem
+collector.server-name: vector.demo.logjet
+upstream.replay: 127.0.0.1:7022
+upstream.mode: keep
+upstream.state-file: ./bridge-grpc-tls.state
+upstream.retry-ms: 1000
+upstream.connect-timeout-ms: 5000
+ingest.protocol: wire
+ingest.listen: 127.0.0.1:7021
+replay.listen: 127.0.0.1:7023

demo/proxy-to-vector/bridge-http-logjetd.conf

@@ -0,0 +1,13 @@
+output: buffer
+buffer.messages: 16
+buffer.keep: 0
+collector.url: http://127.0.0.1:4318/v1/logs
+collector.timeout-ms: 10000
+upstream.replay: 127.0.0.1:7002
+upstream.mode: keep
+upstream.state-file: ./bridge-http.state
+upstream.retry-ms: 1000
+upstream.connect-timeout-ms: 5000
+ingest.protocol: wire
+ingest.listen: 127.0.0.1:7001
+replay.listen: 127.0.0.1:7003

demo/proxy-to-vector/certs/gen-certs.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+CA_PEM="$SCRIPT_DIR/ca.pem"
+CA_KEY="$SCRIPT_DIR/ca.key"
+SERVER_PEM="$SCRIPT_DIR/vector.pem"
+SERVER_KEY="$SCRIPT_DIR/vector.key"
+
+if [ -f "$CA_PEM" ] && [ -f "$CA_KEY" ] && [ -f "$SERVER_PEM" ] && [ -f "$SERVER_KEY" ]; then
+    exit 0
+fi
+
+if ! command -v openssl >/dev/null 2>&1; then
+    echo "missing openssl"
+    exit 1
+fi
+
+tmpdir=$(mktemp -d)
+cleanup() {
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT INT TERM
+
+cat >"$tmpdir/ca.cnf" <<'EOF'
+[req]
+distinguished_name = dn
+x509_extensions = v3_ca
+prompt = no
+
+[dn]
+CN = proxy-to-vector demo CA
+
+[v3_ca]
+basicConstraints = critical,CA:TRUE
+keyUsage = critical,keyCertSign,cRLSign
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+EOF
+
+cat >"$tmpdir/vector.cnf" <<'EOF'
+[req]
+distinguished_name = dn
+req_extensions = v3_req
+prompt = no
+
+[dn]
+CN = vector.demo.logjet
+
+[v3_req]
+basicConstraints = critical,CA:FALSE
+keyUsage = critical,digitalSignature,keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = vector.demo.logjet
+IP.1 = 127.0.0.1
+EOF
+
+openssl req -x509 -newkey rsa:2048 -nodes -keyout "$CA_KEY" -out "$CA_PEM" -days 3650 -config "$tmpdir/ca.cnf" >/dev/null 2>&1
+openssl req -newkey rsa:2048 -nodes -keyout "$SERVER_KEY" -out "$tmpdir/vector.csr" -config "$tmpdir/vector.cnf" >/dev/null 2>&1
+openssl x509 -req -in "$tmpdir/vector.csr" -CA "$CA_PEM" -CAkey "$CA_KEY" -CAcreateserial -out "$SERVER_PEM" -days 3650 -extensions v3_req -extfile "$tmpdir/vector.cnf" >/dev/null 2>&1