Merge pull request #33 from openmoq/tarball-deps

Replace getdeps with tarball and standalone build paths

Author: gmarzot

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  checks: write
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux
+            preset: default
+            build_dir: build
+          - name: asan debug
+            preset: san
+            build_dir: build-san
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Install system dependencies
+        run: bash deps/moxygen/standalone/install-system-deps.sh
+
+      - name: Download moxygen release tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash scripts/setup-deps-tarball.sh
+
+      - name: Configure
+        run: bash scripts/configure.sh ${{ matrix.build_dir }} ${{ matrix.preset }}
+
+      - name: Build
+        run: cmake --build ${{ matrix.build_dir }} -j$(nproc)
+
+      - name: Test
+        env:
+          ASAN_OPTIONS: ${{ matrix.name == 'asan debug' && 'detect_leaks=1:abort_on_error=1' || '' }}
+        run: ctest --test-dir ${{ matrix.build_dir }} --output-on-failure --output-junit test-results.xml
+
+      - name: Publish test results
+        uses: dorny/test-reporter@v1.9.1
+        if: success() || failure()
+        with:
+          name: "test (${{ matrix.name }})"
+          path: ${{ matrix.build_dir }}/test-results.xml
+          reporter: java-junit

.gitignore

@@ -24,3 +24,7 @@ _CPack_Packages/
 
 # OS
 .DS_Store
+
+# IDE / tools
+.vscode/
+.claude/

CMakeLists.txt

@@ -65,7 +65,6 @@ if(ORLY_BUILD_TESTS)
   )
   target_link_libraries(o_rly_relay_test PRIVATE
     o_rly_core
-    moxygen::moqtest_utils
     moxygen::moxygen_events_moq_folly_executor_impl
     GTest::gtest_main
     GTest::gmock

CMakePresets.json

@@ -13,7 +13,8 @@
       "binaryDir": "${sourceDir}/build",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
-        "CMAKE_FIND_LIBRARY_SUFFIXES": ".a"
+        "CMAKE_FIND_LIBRARY_SUFFIXES": ".a",
+        "CMAKE_MODULE_PATH": "${sourceDir}/cmake"
       }
     },
     {
@@ -24,7 +25,8 @@
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Debug",
         "ORLY_ENABLE_SANITIZERS": "ON",
-        "CMAKE_FIND_LIBRARY_SUFFIXES": ".a"
+        "CMAKE_FIND_LIBRARY_SUFFIXES": ".a",
+        "CMAKE_MODULE_PATH": "${sourceDir}/cmake"
       }
     }
   ],

cmake/Findc-ares.cmake

@@ -0,0 +1,23 @@
+# Findc-ares.cmake
+# Wraps system c-ares for platforms where libc-ares-dev does not install
+# cmake config files (e.g. Ubuntu 22.04 with generic cmake >= 3.25 binary).
+
+if(TARGET c-ares::cares)
+  return()
+endif()
+
+find_library(c-ares_LIBRARY NAMES cares)
+find_path(c-ares_INCLUDE_DIR NAMES ares.h)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(c-ares
+  REQUIRED_VARS c-ares_LIBRARY c-ares_INCLUDE_DIR
+)
+
+if(c-ares_FOUND AND NOT TARGET c-ares::cares)
+  add_library(c-ares::cares UNKNOWN IMPORTED)
+  set_target_properties(c-ares::cares PROPERTIES
+    IMPORTED_LOCATION "${c-ares_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${c-ares_INCLUDE_DIR}"
+  )
+endif()

deps/moxygen

@@ -0,0 +1,183 @@
+# Two-Branch Architecture for openmoq/moxygen
+
+## Context
+
+The current single-branch approach has compounding problems: patches modify
+upstream files creating perpetual conflicts, getdeps builds 30+ deps producing
+1.7 GB tarballs, and the sync workflow has grown complex. The standalone build
+(upstream PR #99, merged) eliminates most of this: no manifests, 5 FetchContent
+deps, ~50-150 MB artifacts.
+
+## The Full Flow
+
+### Branches
+
+- **`main`** — Pure upstream mirror. Unmodified copy of latest green upstream
+  commit. No openmoq files. No workflows run. Not the default branch.
+- **`openmoq-main`** (DEFAULT) — Working branch. All openmoq customizations.
+  Developer PRs go here. Artifacts published from here.
+
+### Daily sync cycle
+
+```
+upstream (facebookexperimental/moxygen)
+  │
+  │  1. Sync workflow scans last 20 commits, picks newest green one
+  │
+  ▼
+main (pure mirror)
+  │  2. Fast-forward push: main = upstream green commit
+  │     No workflows fire (all disabled on fork)
+  │
+  │  3. Check: is main already ancestor of openmoq-main?
+  │     YES → done, nothing to merge
+  │     NO  → continue
+  │
+  ▼
+PR: main → openmoq-main
+  │  4. GitHub App bot creates PR (bot identity, not you)
+  │  5. PAT (OMOQ_SYNC_TOKEN) approves PR (different identity, satisfies review)
+  │  6. Auto-merge enabled
+  │
+  │  7. CI fires: standalone build on ubuntu + macOS (PR event)
+  │     PASS → auto-merge fires → merged to openmoq-main
+  │     FAIL → PR stays open, Slack alert, developer resolves
+  │
+  ▼
+openmoq-main (merge lands)
+  │  8. Push to openmoq-main triggers publish-artifacts
+  │  9. Standalone build on all 4 platforms
+  │ 10. Release created: build-<sha> with per-platform tarballs
+  │
+  ▼
+orelay submodule can point to this commit
+```
+
+### Developer workflow
+
+Normal trunk-based git. Developers open PRs against `openmoq-main`. CI runs
+standalone build. Reviews + merge as usual. Changes persist through upstream
+syncs because git merge handles it — once a merge conflict is resolved, the
+merge base advances and the same conflict doesn't recur.
+
+Two classes of local mods:
+- **Upstreamable**: commit on openmoq-main, PR upstream when ready. When upstream
+  absorbs it, next sync merge resolves cleanly (or trivial conflict, resolved once).
+- **Non-upstreamable**: permanent commits on openmoq-main. Git merge carries them
+  forward naturally.
+
+### What runs where
+
+| Branch | Workflows | Trigger |
+|--------|-----------|---------|
+| `main` | **NOTHING** (all upstream workflows disabled) | — |
+| PR → `openmoq-main` | `openmoq-ci.yml` (standalone build) | `pull_request` |
+| `openmoq-main` (push) | `openmoq-publish-artifacts.yml` | `push` |
+| `openmoq-main` (cron) | `openmoq-upstream-sync.yml` | `schedule` / `workflow_dispatch` |
+
+---
+
+## Implementation
+
+### Phase 1: Setup (do first, one-time)
+
+**1a. Create GitHub App** (manual, in GitHub UI)
+- Go to `https://github.com/organizations/openmoq/settings/apps/new`
+- Name: `openmoq-sync-bot` (or similar)
+- Permissions: Contents R/W, Pull requests R/W, Workflows R/W
+- Webhook: off
+- Install on `openmoq/moxygen` only
+- Store: `OMOQ_APP_ID` (variable) + `OMOQ_APP_PRIVATE_KEY` (secret)
+
+**1b. Create branches and set default**
+```bash
+git fetch upstream main
+# Create openmoq-main from current main (has all openmoq files)
+git checkout -b openmoq-main origin/main
+git push -u origin openmoq-main
+# Set as default
+gh api repos/openmoq/moxygen -X PATCH -f default_branch=openmoq-main
+# Reset main to pure upstream
+git push origin upstream/main:refs/heads/main --force-with-lease
+```
+
+**1c. Branch protection**
+- Remove protection from `main` (bot pushes directly)
+- Add protection on `openmoq-main` (required check: standalone build, 1 review)
+
+**1d. Disable upstream workflows** via API
+- getdeps_linux, getdeps_mac, standalone, docker-build-amd64, docker-build-arm64,
+  docker-interop-client — all disabled
+
+**Verification**: `main` = upstream exactly. `openmoq-main` = default with all
+openmoq files. No upstream workflows fire on push to main.
+
+### Phase 2: Sync workflow (rewrite)
+
+`.github/workflows/openmoq-upstream-sync.yml` on `openmoq-main`:
+
+1. Generate app token (`actions/create-github-app-token@v2`)
+2. Checkout `openmoq-main` with app token (for push access)
+3. Scan upstream for newest green commit (reuse existing logic)
+4. Fast-forward `main`: `git push origin $SHA:refs/heads/main`
+5. Check if merge needed (`merge-base --is-ancestor`)
+6. Create PR `main` → `openmoq-main` using app token (bot identity)
+7. Approve PR using `OMOQ_SYNC_TOKEN` (your identity, satisfies review)
+8. Enable auto-merge using app token
+9. Slack on failure
+
+Key simplifications vs current:
+- No candidate branches, no patches, no conflict auto-resolution
+- PR is `main` → `openmoq-main` (if already open, it auto-updates on next push)
+
+### Phase 3: CI + artifact publishing (rewrite)
+
+**`openmoq-ci.yml`** (new) — runs on PRs to `openmoq-main`:
+- Standalone build on ubuntu-22.04 + macos-15
+- `cmake -B _build -S standalone && cmake --build _build && ctest`
+
+**`openmoq-publish-artifacts.yml`** (rewrite) — runs on push to `openmoq-main`:
+- 4-platform matrix (ubuntu, macos, bookworm-amd64, bookworm-arm64)
+- `cmake -B _build -S standalone -DCMAKE_INSTALL_PREFIX=install/`
+- `cmake --build _build && cmake --install _build`
+- Package: gather install prefix + any needed headers into tarball
+- Release: `create-release.sh` (unchanged)
+
+**Headers**: `cmake --install` should install headers for all deps (each
+FetchContent dep has its own install rules). If moxygen's own headers aren't
+covered, the packaging script gathers them from source. This is a packaging
+concern — we verify during Phase 3 and handle in the collection script.
+
+**New `collect-artifacts-standalone.sh`**: takes `--install-prefix` and
+`--src-dir`, strips debug symbols, packages tarball. Much simpler than current
+getdeps-based version.
+
+### Phase 4: Cleanup
+
+- Delete `openmoq/patches/` (no longer needed)
+- Delete/archive old `collect-artifacts.sh`
+- Update `openmoq/README.md` and `GITHUB_WORKFLOW.md`
+- Clean up stale branches
+- Update o-rly `.gitmodules` to track `openmoq-main`
+
+### Phase 5: End-to-end verification
+
+1. Trigger sync → main advances → PR created (bot) → approved (you) → CI passes → merged
+2. Publish-artifacts fires → standalone build → tarball ~50-150 MB → release
+3. Conflict test: local mod on openmoq-main + upstream touches same file → PR open → Slack → resolve
+
+---
+
+## Files
+
+| File | Action | Phase |
+|------|--------|-------|
+| `.github/workflows/openmoq-upstream-sync.yml` | Rewrite for two-branch + app token | 2 |
+| `.github/workflows/openmoq-ci.yml` | Create (standalone build CI) | 3 |
+| `.github/workflows/openmoq-publish-artifacts.yml` | Rewrite for standalone build | 3 |
+| `openmoq/scripts/collect-artifacts-standalone.sh` | Create | 3 |
+| `openmoq/scripts/collect-artifacts.sh` | Delete | 4 |
+| `openmoq/patches/` | Delete directory | 4 |
+| `openmoq/README.md` | Update | 4 |
+| `o-rly/design/GITHUB_WORKFLOW.md` | Update | 4 |
+| `o-rly/.gitmodules` | Track `openmoq-main` | 4 |

design/GITHUB_WORKFLOW.md

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# setup-deps-standalone.sh — Build moxygen + deps from source (standalone/FetchContent).
+#
+# Uses deps/moxygen/standalone/CMakeLists.txt which fetches Meta OSS deps
+# (folly, fizz, wangle, mvfst, proxygen) via FetchContent and builds them
+# as static libraries alongside moxygen. Installs everything to .scratch/
+# moxygen-install and writes cmake_prefix_path.txt for configure.sh.
+#
+# This is the "deep" build — slower first time but fully self-contained.
+# Subsequent builds are incremental (cmake only rebuilds what changed).
+#
+# Usage:
+#   ./scripts/setup-deps-standalone.sh
+#
+# System deps required (Ubuntu):
+#   deps/moxygen/standalone/install-system-deps.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+SCRATCH="${ORLY_SCRATCH_PATH:-${PROJECT_ROOT}/.scratch}"
+MOXYGEN_DIR="${PROJECT_ROOT}/deps/moxygen"
+STANDALONE_SRC="${MOXYGEN_DIR}/standalone"
+BUILD_DIR="${SCRATCH}/standalone-build"
+INSTALL_DIR="${SCRATCH}/moxygen-install"
+
+if [[ ! -e "$MOXYGEN_DIR/.git" ]]; then
+    echo "Error: deps/moxygen submodule not initialized." >&2
+    echo "  Run: git submodule update --init" >&2
+    exit 1
+fi
+
+if [[ ! -f "${STANDALONE_SRC}/CMakeLists.txt" ]]; then
+    echo "Error: standalone/CMakeLists.txt not found in moxygen submodule" >&2
+    exit 1
+fi
+
+NPROC=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
+
+echo "==> Configuring standalone moxygen build..."
+cmake -S "$STANDALONE_SRC" -B "$BUILD_DIR" \
+    -G Ninja \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+    -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+    -DBUNDLE_DEPS=ON \
+    -DBUILD_TESTS=OFF \
+    -DBUILD_SHARED_LIBS=OFF
+
+echo "==> Building ($NPROC jobs)..."
+cmake --build "$BUILD_DIR" -j"$NPROC"
+
+echo "==> Installing to $INSTALL_DIR..."
+rm -rf "$INSTALL_DIR"
+cmake --install "$BUILD_DIR"
+
+# ── Write cmake_prefix_path.txt ───────────────────────────────────────────────
+
+mkdir -p "$SCRATCH"
+echo "$INSTALL_DIR" > "${SCRATCH}/cmake_prefix_path.txt"
+echo "standalone" > "${SCRATCH}/deps-mode"
+
+NLIBS=$(find "$INSTALL_DIR/lib" -name '*.a' 2>/dev/null | wc -l)
+echo "==> Done: $NLIBS static libs in $INSTALL_DIR"