Merge pull request #1148 from presidojay1/fix/issues-778-851-779-775

Fix enum discriminant collision, fuzz multisig, trace outbox boundary, reproducible builds, SLOs

Author: joelpeace48-cell

.github/workflows/contract-fuzzing.yml

@@ -80,6 +80,31 @@ jobs:
 
           echo "Fuzzing completed for ${{ matrix.contract }} contract"
 
+      - name: Install nightly toolchain + cargo-fuzz for libFuzzer targets (rewards only)
+        if: matrix.contract == 'rewards'
+        continue-on-error: true
+        run: |
+          rustup toolchain install nightly
+          cargo install cargo-fuzz --locked
+
+      - name: Run libFuzzer targets (issue #851: fuzz_multisig, plus fuzz_balance)
+        if: matrix.contract == 'rewards'
+        continue-on-error: true
+        working-directory: contracts/rewards
+        run: |
+          echo "Running cargo-fuzz libFuzzer targets for ${FUZZ_DURATION} seconds each..."
+          cargo +nightly fuzz run fuzz_balance -- -max_total_time=${FUZZ_DURATION} || echo "fuzz_balance completed/found an issue"
+          cargo +nightly fuzz run fuzz_multisig -- -max_total_time=${FUZZ_DURATION} || echo "fuzz_multisig completed/found an issue"
+
+      - name: Upload libFuzzer crash artifacts (rewards only)
+        if: matrix.contract == 'rewards' && always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: libfuzzer-artifacts-rewards-${{ github.run_number }}
+          path: contracts/rewards/fuzz/artifacts/
+          if-no-files-found: ignore
+          retention-days: 30
+
       - name: Check for regression files
         run: |
           if [ -d "contracts/${{ matrix.contract }}/proptest-regressions" ] && [ "$(ls -A contracts/${{ matrix.contract }}/proptest-regressions)" ]; then

backend/package.json

@@ -29,6 +29,7 @@
     "@opentelemetry/instrumentation-http": "^0.55.0",
     "@opentelemetry/resources": "^1.28.0",
     "@opentelemetry/sdk-node": "^0.55.0",
+    "@opentelemetry/sdk-trace-base": "^1.28.0",
     "@opentelemetry/semantic-conventions": "^1.28.0",
     "@stellar/stellar-sdk": "^14.0.0",
     "better-sqlite3": "^11.10.0",

backend/src/services/outboxService.js

@@ -10,8 +10,16 @@
  *   const relay = new OutboxRelay(db, handlers, { logger });
  *   relay.start();     // begin polling
  *   relay.stop();      // graceful shutdown
+ *
+ * Tracing (issue #778): `writeOutbox` captures the enqueuing request's trace
+ * context and stores it alongside the payload; `_deliver` re-establishes it
+ * as the parent of a new `outbox.deliver` span, so a single trace shows the
+ * full request → job path even though delivery happens on a later,
+ * unrelated poll tick.
  */
 
+import { captureTraceparent, linkedSpan } from '../tracing.js';
+
 const DEFAULT_POLL_INTERVAL_MS = 5_000;
 const DEFAULT_LOCK_DURATION_MS = 60_000;
 const DEFAULT_MAX_ATTEMPTS = 5;
@@ -34,11 +42,18 @@ export function writeOutbox(db, eventType, payload, opts = {}) {
   const deliverAfter =
     delayMs > 0 ? new Date(Date.now() + delayMs).toISOString() : new Date().toISOString();
 
+  // Stash the enqueuing request's trace context inside the stored envelope
+  // (issue #778) rather than adding a DB column for it — `_traceparent` is a
+  // reserved key on the envelope, never on the caller's own payload, so it
+  // round-trips through the existing `payload TEXT` column with no schema
+  // migration.
+  const envelope = { payload, _traceparent: captureTraceparent() };
+
   const stmt = db.prepare(`
     INSERT INTO outbox (event_type, payload, partition_key, deliver_after)
     VALUES (?, ?, ?, ?)
   `);
-  const result = stmt.run(eventType, JSON.stringify(payload), partitionKey, deliverAfter);
+  const result = stmt.run(eventType, JSON.stringify(envelope), partitionKey, deliverAfter);
   return result.lastInsertRowid;
 }
 
@@ -122,19 +137,37 @@ export class OutboxRelay {
   async _deliver(row) {
     const handler = this.handlers[row.event_type];
     let payload;
+    let traceparent = null;
     try {
-      payload = JSON.parse(row.payload);
+      const parsed = JSON.parse(row.payload);
+      // Rows written before issue #778 store the raw payload directly;
+      // rows written since wrap it as `{ payload, _traceparent }`. Detect by
+      // the reserved `_traceparent` key rather than assuming every row is
+      // the new shape, so already-queued (pre-deploy) rows still deliver.
+      if (parsed && typeof parsed === 'object' && Object.prototype.hasOwnProperty.call(parsed, '_traceparent')) {
+        payload = parsed.payload;
+        traceparent = parsed._traceparent;
+      } else {
+        payload = parsed;
+      }
     } catch {
       this._markFailed(row.id, 'invalid JSON payload');
       return;
     }
 
     try {
-      if (handler) {
-        await handler(payload);
-      } else {
-        this.logger.warn({ eventType: row.event_type }, 'no outbox handler registered');
-      }
+      await linkedSpan(
+        traceparent,
+        'outbox.deliver',
+        { 'outbox.event_type': row.event_type, 'outbox.id': row.id, 'outbox.attempts': row.attempts },
+        async () => {
+          if (handler) {
+            await handler(payload);
+          } else {
+            this.logger.warn({ eventType: row.event_type }, 'no outbox handler registered');
+          }
+        },
+      );
       this.db
         .prepare(`UPDATE outbox SET status = 'delivered', locked_until = NULL WHERE id = ?`)
         .run(row.id);

backend/src/tracing.js

@@ -28,7 +28,13 @@
  * auto-instrumentation patches miss the loaded modules.
  */
 
-import { trace, SpanStatusCode, context as otelContext } from '@opentelemetry/api';
+import {
+  trace,
+  SpanStatusCode,
+  SpanKind,
+  context as otelContext,
+  propagation,
+} from '@opentelemetry/api';
 
 let sdkInstance = null;
 
@@ -275,6 +281,64 @@ export function traceparentMiddleware() {
 /** Headers to expose so a browser fetch can read the traceparent. */
 export const TRACING_EXPOSED_HEADERS = ['traceparent'];
 
+/**
+ * Capture the currently active span's context as a W3C `traceparent`
+ * string, or `null` if there is no active span (issue #778).
+ *
+ * The transactional outbox pattern (`outboxService.js`) breaks OTel's
+ * automatic in-process context propagation: a row is written now, inside
+ * the HTTP request's trace, but delivered later by a completely separate
+ * poll loop tick with no ambient span. Persisting the traceparent string
+ * alongside the outbox row is what lets `linkedSpan()` below re-establish
+ * that parent/child relationship once the relay picks the row up.
+ */
+export function captureTraceparent() {
+  const span = trace.getSpan(otelContext.active());
+  if (!span) return null;
+  const ctx = span.spanContext();
+  const flags = ctx.traceFlags.toString(16).padStart(2, '0');
+  return `00-${ctx.traceId}-${ctx.spanId}-${flags}`;
+}
+
+/**
+ * Run `fn` inside a new span that is a *child* of the trace identified by
+ * `traceparent` (as produced by `captureTraceparent()`), even though this
+ * call is happening on a later event-loop tick / different logical
+ * "request" than the one that created it — the async-boundary case issue
+ * #778 asks for (outbox relay, background jobs).
+ *
+ * Falls back to a plain, unlinked `withSpan()` when `traceparent` is
+ * missing or malformed, so a job enqueued before this feature existed (no
+ * stored traceparent) still gets traced, just without a parent link.
+ */
+export async function linkedSpan(traceparent, name, attributes, fn) {
+  if (!traceparent) {
+    return withSpan(name, attributes, fn);
+  }
+
+  const parentContext = propagation.extract(otelContext.active(), { traceparent });
+  return otelContext.with(parentContext, () => {
+    const tracer = trace.getTracer('trivela-backend');
+    return tracer.startActiveSpan(
+      name,
+      { attributes, kind: SpanKind.CONSUMER },
+      async (span) => {
+        try {
+          const result = await fn(span);
+          span.setStatus({ code: SpanStatusCode.OK });
+          return result;
+        } catch (err) {
+          span.recordException(err);
+          span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+          throw err;
+        } finally {
+          span.end();
+        }
+      },
+    );
+  });
+}
+
 /** Graceful shutdown hook — flush exporter on SIGTERM. */
 export async function shutdownTracing() {
   if (sdkInstance && typeof sdkInstance.shutdown === 'function') {

backend/src/tracing.test.js

@@ -1,3 +1,94 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { trace, context as otelContext } from '@opentelemetry/api';
+import {
+  BasicTracerProvider,
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} from '@opentelemetry/sdk-trace-base';
+import { captureTraceparent, linkedSpan } from './tracing.js';
+
+/**
+ * Trace assertion test for issue #778: confirms `captureTraceparent()` +
+ * `linkedSpan()` correctly re-link a span across an async boundary — the
+ * exact shape of the outbox relay's write-now/deliver-later gap, where
+ * OTel's automatic context propagation can't reach on its own since the
+ * delivery happens on a completely separate event-loop tick with no
+ * ambient span.
+ */
+
+function setupInMemoryTracing() {
+  const exporter = new InMemorySpanExporter();
+  const provider = new BasicTracerProvider({
+    spanProcessors: [new SimpleSpanProcessor(exporter)],
+  });
+  trace.setGlobalTracerProvider(provider);
+  return { provider, exporter };
+}
+
+test('linkedSpan re-establishes parent/child across an async boundary', async () => {
+  const { provider, exporter } = setupInMemoryTracing();
+  try {
+    const tracer = trace.getTracer('test');
+
+    // Simulate the HTTP request that enqueues the outbox row: start a span,
+    // capture its traceparent (what writeOutbox stores alongside the row),
+    // then end the span — mirroring the request finishing well before the
+    // relay ever picks the row up.
+    let traceparent;
+    let parentSpanId;
+    let parentTraceId;
+    await tracer.startActiveSpan('http.request', async (parentSpan) => {
+      const ctx = parentSpan.spanContext();
+      parentSpanId = ctx.spanId;
+      parentTraceId = ctx.traceId;
+      traceparent = captureTraceparent();
+      parentSpan.end();
+    });
+
+    assert.ok(traceparent, 'expected a traceparent to be captured from the active span');
+
+    // Simulate the relay's later, unrelated poll tick: no ambient span here
+    // at all — linkedSpan must reconstruct the parent purely from the
+    // stored traceparent string.
+    await otelContext.with(otelContext.active(), async () => {
+      await linkedSpan(traceparent, 'outbox.deliver', { 'outbox.event_type': 'test.event' }, async () => {
+        // handler body — nothing to do for this assertion.
+      });
+    });
+
+    await provider.forceFlush();
+    const spans = exporter.getFinishedSpans();
+    const deliverSpan = spans.find((s) => s.name === 'outbox.deliver');
+
+    assert.ok(deliverSpan, 'expected an outbox.deliver span to have been recorded');
+    assert.equal(
+      deliverSpan.parentSpanId,
+      parentSpanId,
+      'outbox.deliver span must be a child of the request span that enqueued it',
+    );
+    assert.equal(
+      deliverSpan.spanContext().traceId,
+      parentTraceId,
+      'outbox.deliver span must share the same trace id as the request that enqueued it',
+    );
+  } finally {
+    await provider.shutdown();
+  }
+});
+
+test('linkedSpan still traces (unlinked) when no traceparent is available', async () => {
+  const { provider, exporter } = setupInMemoryTracing();
+  try {
+    await linkedSpan(null, 'outbox.deliver', {}, async () => {});
+    await provider.forceFlush();
+    const spans = exporter.getFinishedSpans();
+    const deliverSpan = spans.find((s) => s.name === 'outbox.deliver');
+    assert.ok(deliverSpan, 'expected a span even without a traceparent to link to');
+    assert.equal(deliverSpan.parentSpanId, undefined, 'span should have no parent when unlinked');
+  } finally {
+    await provider.shutdown();
+  }
 /**
  * Tests for distributed tracing across async boundaries
  * 

contracts/rewards/fuzz/Cargo.lock

@@ -15,9 +15,16 @@ cargo-fuzz = true
 libfuzzer-sys = "0.4"
 trivela-rewards-contract = { path = "..", features = ["testutils"] }
 soroban-sdk = { version = "25.1", features = ["testutils"] }
+ed25519-dalek = "2"
 
 [[bin]]
 name = "fuzz_balance"
 path = "fuzz_targets/fuzz_balance.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "fuzz_multisig"
+path = "fuzz_targets/fuzz_multisig.rs"
+test = false
+doc = false