fix: refactor and splitting modules

Author: Lundez

.fluxel/config.json

@@ -0,0 +1 @@
+{"backend":"local","format_version":1,"dataset_root":"/Users/londogard/git/strand","default_branch":"main","identity":"blake3","transfer_backend":null}

.fluxel/refs/HEAD

@@ -0,0 +1 @@
+refs/heads/main

.fluxel/refs/heads/main

@@ -0,0 +1 @@
+{"commit_id": null, "version_token": "1783593278287833390-0"}

.fluxel/refs/heads/main.json

@@ -0,0 +1,115 @@
+name: CI
+
+on:
+    push:
+    pull_request:
+
+jobs:
+    test-and-build:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Check out repository
+              uses: actions/checkout@v4
+
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.12"
+
+            - name: Set up uv
+              uses: astral-sh/setup-uv@v4
+
+            - name: Install dependencies
+              run: uv sync --group dev
+
+            - name: Reject macOS junk files
+              run: |
+                  if git ls-files | grep -E '(^__MACOSX/|(^|/)\.DS_Store$|(^|/)\._)'; then
+                    echo "Remove macOS archive/junk files before merging."
+                    exit 1
+                  fi
+
+            - name: Run non-integration tests
+              run: uv run pytest tests -m "not integration"
+
+            - name: Smoke-test CLI help
+              run: uv run fluxel --help
+
+            - name: Build package
+              run: uv build
+
+            - name: Clean-tree check
+              run: |
+                  if [ -n "$(git status --porcelain)" ]; then
+                    echo "Working tree is dirty after build:"
+                    git status
+                    git diff
+                    exit 1
+                  fi
+
+            - name: Upload wheel artifact
+              uses: actions/upload-artifact@v4
+              with:
+                  name: wheel
+                  path: dist/*.whl
+                  if-no-files-found: error
+
+    s3-integration:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Check out repository
+              uses: actions/checkout@v4
+
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.12"
+
+            - name: Set up uv
+              uses: astral-sh/setup-uv@v4
+
+            - name: Install dependencies
+              run: uv sync --group dev
+
+            - name: Run S3 integration tests against MiniStack
+              run: bash scripts/run_s3_integration.sh
+
+    isolated-install-smoke:
+        needs: [test-and-build]
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Set up Python
+              uses: actions/setup-python@v5
+              with:
+                  python-version: "3.12"
+
+            - name: Download wheel artifact
+              uses: actions/download-artifact@v4
+              with:
+                  name: wheel
+                  path: dist
+
+            - name: Isolated install and smoke
+              run: |
+                  set -euo pipefail
+                  WHEEL=$(ls dist/*.whl | head -1)
+                  python -m venv /tmp/smoke-venv
+                  /tmp/smoke-venv/bin/pip install "$WHEEL"
+                  echo "=== fluxel --help ==="
+                  /tmp/smoke-venv/bin/fluxel --help
+                  echo "=== fluxel --version ==="
+                  /tmp/smoke-venv/bin/fluxel --version 2>/dev/null || true
+                  echo "=== basic commit smoke ==="
+                  TMPDIR=$(mktemp -d)
+                  echo "smoke-test-content" > "$TMPDIR/smoke.txt"
+                  /tmp/smoke-venv/bin/fluxel commit --repo "$TMPDIR" -m "smoke" > /dev/null
+                  echo "Isolated install smoke test passed."
+
+    release-gate:
+        needs: [test-and-build, s3-integration, isolated-install-smoke]
+        runs-on: ubuntu-latest
+        steps:
+            - run: echo "All release gates passed."

workflows/bench.txt .github/workflows/bench.txtworkflows/bench.txt renamed to .github/workflows/bench.txt

@@ -148,6 +148,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+.pyrefly_cache/
 
 # pytype static type analyzer
 .pytype/

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+# Pre-launch TODOs
+
+This list is deliberately launch-oriented rather than a feature roadmap.
+
+## P0 — complete before users can share repositories
+
+- [X] Make `push` and `pull` reject divergent history and check the CAS result.
+  They currently copy objects and then move the destination branch without
+  proving that the destination head is an ancestor; they also report success
+  when `compare_and_set_branch_ref` returns `False`. CAS prevents a race, not
+  a destructive non-fast-forward update. Add two-client divergence and
+  concurrent-update tests for both directions.
+- [x] Add a release gate that runs: non-integration tests, real S3 integration,
+  an isolated install of the built wheel, CLI smoke tests, and a clean-tree
+  check. Implemented in `.github/workflows/ci.yml`:
+  `test-and-build` (non-integration tests + build + clean-tree + artifact upload),
+  `s3-integration` (real S3 against MiniStack),
+  `isolated-install-smoke` (wheel install in fresh venv + CLI smoke),
+  and `release-gate` (unified gate depending on all three).
+  The `release-gate` job can be set as the single required status check
+  in branch-protection rules.
+- [x] Decide and document the durability contract for `--identity meta`.
+  Source URIs can change after a metadata-only import; label such revisions as
+  unverifiable until `verify`, warn in CLI output, and document the required
+  source-retention policy.
+  Implemented:
+  - `ManifestEntry.is_verified` property distinguishes blob-backed from
+    metadata-only entries.
+  - CLI warns to stderr on `add --identity meta`, `commit` with meta config,
+    and `verify` when unverifiable entries remain.
+  - Durability contract and source-retention policy documented in README
+    under "Identity Modes → Durability contract for --identity meta".
+- [x] Publish operator runbooks for S3 IAM, encryption, bucket versioning,
+  lifecycle/retention, backups, lock recovery, and incident recovery. Add a
+  supported lock inspection/cleanup command before relying on shared S3 repos.
+  Implemented:
+  - `docs/operator-runbook.md` covers IAM policy, SSE-S3/KMS encryption,
+    bucket versioning, lifecycle rules, backup/restore strategies, lock
+    recovery procedures, and five incident recovery scenarios.
+  - `fluxel lock list` and `fluxel lock cleanup` CLI commands with `--json`,
+    `--force`, and `--repo` flags.
+  - `S3RepositoryStore.list_branch_locks()`, `.branch_lock_info()`, and
+    `.force_release_branch_lock()` plus `FluxelRepository.list_locks()`,
+    `.lock_info()`, `.force_release_lock()`, and `.lock_timeout_seconds`.
+
+## P1 — strongly recommended for the first public release
+
+- [ ] Make the configured type-check gate useful. `uv run pyrefly check`
+  currently reports 89 errors (including exported names, protocol typing, and
+  filesystem return types), while the config keys emit warnings. Either fix the
+  errors or scope/configure the check deliberately, then add it to CI.
+- [ ] Update the README's MVP status: it says remote sync CLI is not wired,
+  but `push`, `pull`, and `fetch` now exist. Document their safety semantics,
+  supported remotes, and recovery workflow.
+- [ ] Replace whole-object sync copies with streamed transfers. The current
+  sync helper reads each complete blob into memory and writes a temporary file,
+  which makes large-object sync memory- and disk-heavy.
+- [ ] Change fsspec dataset resolution to fail for an unknown dataset instead
+  of falling back to the current directory. A typo can otherwise read from an
+  unintended local repository.
+- [ ] Test the supported-version matrix: Python 3.11 and 3.12, current fsspec,
+  AWS S3, and the configured S3-compatible endpoint. Include interrupted
+  transfer, stale-lock, access-denied, and corrupted-object scenarios.
+- [ ] Add privacy/security release checks: dependency/vulnerability scan,
+  license review of runtime dependencies, documentation on secrets/credential
+  handling, and a statement of telemetry behavior.
+
+## P2 — early operational follow-ups
+
+- [ ] Add remote configuration (`remote add`) and clone; requiring a full S3
+  URI on every sync is error-prone.
+- [ ] Add garbage collection with a dry-run and retention policy for unreachable
+  blobs/manifests.
+- [ ] Add machine-readable release notes, versioning/migration policy, and a
+  reproducible PyPI publishing workflow.
+
+
+----
+
+Architecture?
+
+  CLI / fsspec
+      ↓
+  Application use cases
+  (commit, add, verify, sync, restore, merge)
+      ↓
+  Domain
+  (commits, manifests, paths, identity, commit graph, errors)
+      ↓
+  Ports
+  (RepositoryStore, ClientState, Workspace, BlobTransfer)
+      ↓
+  Adapters
+  (local filesystem, S3, fsspec, CLI formatting)
+
+**Concretely:**
+* Keep ManifestEntry, path validation, hashing rules, commit graph ancestry, and domain errors pure—no Path, boto3, or CLI imports.
+* Make RepositoryStore the shared-object port, ClientState the local mutable-state port, and add a streaming BlobTransfer port.
+* Move each command into a small use case with explicit input/result objects: CommitUseCase, SyncUseCase, RestoreUseCase, etc.
+* Make ref updates go through one operation such as advance_ref(expected_head, new_head). The application layer must prove fast-forward ancestry and must treat a failed CAS as a conflict.
+* Keep immutable-object publication separate from the final ref update: upload blobs → manifest → commit → CAS branch ref.
+* Add one contract-test suite that every RepositoryStore implementation must pass; run it against local storage and S3-compatible storage.
+* Keep fsspec as a thin read adapter, not part of repository business logic.

.gitignore

@@ -34,6 +34,8 @@ Fluxel is intentionally in MVP mode.
 - Commit snapshots over a dataset root (`fluxel commit`).
 - Repository URI support via `--repo <path|s3://bucket/prefix>` and `open_repository(...)`.
 - Repository initialization (`fluxel init`) with local or S3 backend.
+- Remote sync workflow (`fluxel fetch`, `fluxel pull`, `fluxel push`).
+- Operator-facing S3 lock inspection and cleanup (`fluxel lock list`, `fluxel lock cleanup`).
 - Streaming S3 imports (`fluxel import`) with `blake3` or metadata identity modes.
 - Branch-scoped staging workflow (`fluxel add`, `fluxel rm`, `fluxel status`, `fluxel commit --staged`).
 - Incremental ingress paths that preserve existing manifest entries while adding only new content metadata/blobs.
@@ -49,11 +51,6 @@ Fluxel is intentionally in MVP mode.
 - Local + S3 storage backend abstractions available in code.
 - Human-readable output by default; `--json` flag on all commands for programmatic use.
 
-### Not Wired Yet
-
-- No remote sync CLI (`push/pull/fetch`) yet.
-- No operator-facing S3 lock inspection or cleanup command.
-
 ## Technical Stack
 
 - Python 3.11+
@@ -185,6 +182,36 @@ In `--identity meta` snapshots, Fluxel reads from `source_uri` when no canonical
 
 This is useful for large bootstrap imports where strong content verification can be deferred.
 
+### Durability contract for `--identity meta`
+
+Metadata-only (`--identity meta`) revisions are **unverifiable**: the entry's
+identity is derived from path and size, not from content bytes. Until you run
+`fluxel verify`, Fluxel cannot prove that the content at `source_uri` matches
+what was originally imported.
+
+**Warnings.** The CLI emits a warning to stderr whenever you stage or commit
+with `--identity meta`, and after `verify` reports how many unverifiable entries
+remain.
+
+**Source-retention policy.** Because metadata-only entries have no canonical
+blob, you **must** retain the source objects at their original `source_uri`
+until the entry has been promoted via `fluxel verify`. If a source object is
+deleted, overwritten, or moved before verification, the corresponding manifest
+entry becomes irrecoverable — no content can be read and no hash can be
+validated.
+
+**Promotion to verifiable.** Run `fluxel verify` to read every metadata-only
+entry's source blob, compute a Blake3 content hash, store the canonical blob,
+and rewrite the manifest entry in `blake3` mode. After promotion the source
+retention requirement is lifted for those entries.
+
+**Lifecycle summary:**
+
+| State | `identity_mode` | `blob_hash` | Can read? | Can prove integrity? | Source required? |
+|---|---|---|---|---|---|
+| Metadata-only | `meta` | `null` | ✅ (from `source_uri`) | ❌ | ✅ |
+| Verified | `blake3` | hash | ✅ (from `blobs/`) | ✅ | ❌ |
+
 ## Verify Command
 
 `fluxel verify` promotes metadata-only (`--identity meta`) manifest entries into canonical `blake3` blob-backed entries: