From fad800b687ce94f291c3630ca1a33e59d8305729 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 29 May 2026 15:24:58 +0400 Subject: [PATCH 1/9] Storacha migration tool --- cmd/curio/storacha-migration.go | 669 +++++++++++++++++++++++++ cmd/curio/storacha-migration_test.go | 718 +++++++++++++++++++++++++++ cmd/curio/toolbox.go | 1 + documentation/en/curio-cli/curio.md | 16 + 4 files changed, 1404 insertions(+) create mode 100644 cmd/curio/storacha-migration.go create mode 100644 cmd/curio/storacha-migration_test.go diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go new file mode 100644 index 000000000..dc6d999b4 --- /dev/null +++ b/cmd/curio/storacha-migration.go @@ -0,0 +1,669 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ipfs/go-cid" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/cmd/curio/internal/translations" + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/parkpiece" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" +) + +const minSaveCacheSize = abi.PaddedPieceSize(uint64(32 * 1024 * 1024)) +const serviceName = "public" +const storachaMigrationDataURL = "storacha-migration" + +type importPiecesOutput struct { + Count int `json:"count"` + Pieces []string `json:"pieces"` +} + +type storachaImportResult struct { + Imported bool + PieceCID string +} + +type storachaParkedPieceState struct { + ID int64 + UseStagedFile bool + AlreadyComplete bool +} + +type storachaFinalRecoveryRow struct { + ID int64 `db:"id"` + PieceCID string `db:"piece_cid"` + PaddedSize int64 `db:"piece_padded_size"` + RawSize int64 `db:"piece_raw_size"` +} + +var importPiecesCmd = &cli.Command{ + Name: "import-pieces", + Usage: translations.T("Imports already existing pieces from storage to piece park system"), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "source", + Usage: "path to store the pieces from", + }, + &cli.StringFlag{ + Name: "target", + Usage: "path to piece storage directory in Curio's attached permanent storage", + }, + &cli.IntFlag{ + Name: "batch-size", + Usage: "number of pieces to move to permanent storage", + Value: 20, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + + /* + How this migration works: + + The source directory has Storacha CAR files named by piece CID v2. Curio's + piece park stores the same bytes under a parked_pieces id, so the final + filename becomes s-t00-. That means the migration must + first learn or create the parked_pieces row before it knows the final + filename. + + Every fresh source file is moved to target/storacha-staging first. That + staging move is the first retry boundary: once the file leaves source, a + later run can still find it and finish the DB/ref/final-file work through + importStagedStorachaPiece. + + importStagedStorachaPiece owns the normal path: + 1. Parse the staged filename to get the piece info. + 2. In one DB transaction, create or reuse the parked_pieces row. + 3. If the row is incomplete and has a live harmony task, leave the staged + file alone because that task owns the normal download/write path. + 4. Otherwise, ensure exactly one storacha-migration parked_piece_refs row + and exactly one pdp_piecerefs row for that parked ref. + 5. After the transaction commits, move the staged file to s-t00-, + declare it in sector_location, then mark parked_pieces.complete=true. + + recoverFinalStorachaPieces exists for the one point where staging can no + longer help: if a previous run already renamed the file to s-t00- and + then crashed before declaring the file or marking the parked piece + complete. At that point there is no staged filename left to process, so + recovery starts from the DB row/ref and checks whether the final file is + already present. + */ + + if cctx.String("source") == "" || cctx.String("target") == "" { + return fmt.Errorf("source and target both are required") + } + + sourcePath, err := homedir.Expand(cctx.String("source")) + if err != nil { + return xerrors.Errorf("expanding source path: %w", err) + } + + targetPath, err := homedir.Expand(cctx.String("target")) + if err != nil { + return xerrors.Errorf("expanding target path: %w", err) + } + targetPath, err = filepath.Abs(filepath.Clean(targetPath)) + if err != nil { + return xerrors.Errorf("resolving target path: %w", err) + } + + dep, err := deps.GetDepsCLI(ctx, cctx) + if err != nil { + return err + } + + out, err := runImportPieces(ctx, dep, sourcePath, targetPath, cctx.Int("batch-size")) + if err != nil { + return err + } + + err = PrintJson(out) + if err != nil { + return err + } + + return nil + }, +} + +func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { + var out importPiecesOutput + + targetPath, err := filepath.Abs(filepath.Clean(targetPath)) + if err != nil { + return out, xerrors.Errorf("resolving target path: %w", err) + } + + storageID, err := pieceStorageID(ctx, dep, targetPath) + if err != nil { + return out, err + } + + staging := filepath.Join(targetPath, "storacha-staging") + existing, err := os.ReadDir(staging) + if err != nil { + if !os.IsNotExist(err) { + return out, xerrors.Errorf("reading staging directory: %w", err) + } + + err = os.MkdirAll(staging, 0755) + if err != nil { + return out, xerrors.Errorf("creating staging directory: %w", err) + } + } + + if len(existing) > 0 { + // Staging is the retry boundary: a file here was already moved out of + // source, but DB refs or final piece declaration may not be complete. + for _, entry := range existing { + if out.Count >= batchSize { + break + } + if entry.IsDir() { + continue + } + + result, err := importStagedStorachaPiece(ctx, dep, storageID, targetPath, filepath.Join(staging, entry.Name())) + if err != nil { + return out, err + } + if result.Imported { + out.Pieces = append(out.Pieces, result.PieceCID) + out.Count++ + } + } + } + + // This covers the crash point after staging was renamed to the final + // piece path, but before sector_location/parked_pieces was finalized. + err = recoverFinalStorachaPieces(ctx, dep, storageID, targetPath, &out, batchSize) + if err != nil { + return out, err + } + + // Open the directory and read all .car files in it. No need to care about subdirectories + entries, err := os.ReadDir(sourcePath) + if err != nil { + return out, xerrors.Errorf("reading directory: %w", err) + } + + for _, entry := range entries { + if out.Count >= batchSize { + break + } + if entry.IsDir() { + continue + } + + if !strings.HasSuffix(entry.Name(), ".car") { + continue + } + + _, ok, err := storachaPieceInfoFromFileName(entry.Name()) + if err != nil { + return out, err + } + if !ok { + continue + } + + oldPath := filepath.Join(sourcePath, entry.Name()) + stagingPath := filepath.Join(staging, entry.Name()) + if _, err := os.Stat(stagingPath); err == nil { + return out, xerrors.Errorf("staging file already exists: %s", stagingPath) + } else if !os.IsNotExist(err) { + return out, xerrors.Errorf("checking staging file: %w", err) + } + + // New imports also go through staging, so the same recovery code can + // resume after a crash at any later step. + err = os.Rename(oldPath, stagingPath) + if err != nil { + return out, xerrors.Errorf("renaming old path: %w", err) + } + + result, err := importStagedStorachaPiece(ctx, dep, storageID, targetPath, stagingPath) + if err != nil { + return out, err + } + if result.Imported { + out.Pieces = append(out.Pieces, result.PieceCID) + out.Count++ + } + } + + return out, nil +} + +// pieceStorageID maps the user-provided target directory back to Curio's +// storage id. The target flag is expected to point at an attached storage +// path's "piece" directory, because StorageDeclareSector needs the parent +// storage id when it writes sector_location. +func pieceStorageID(ctx context.Context, dep *deps.Deps, targetPath string) (storiface.ID, error) { + localPaths, err := dep.LocalStore.Local(ctx) + if err != nil { + return "", xerrors.Errorf("listing local storage paths: %w", err) + } + + for _, localPath := range localPaths { + piecePath := filepath.Join(localPath.LocalPath, storiface.FTPiece.String()) + piecePath, err = filepath.Abs(filepath.Clean(piecePath)) + if err != nil { + return "", xerrors.Errorf("resolving piece path %s: %w", piecePath, err) + } + if piecePath == targetPath { + return localPath.ID, nil + } + } + + return "", xerrors.Errorf("target %s is not the piece directory of an attached local storage path", targetPath) +} + +// storachaPieceInfoFromFileName accepts only Storacha migration inputs named +// .car. Invalid names or unparsable CIDs are skipped; valid names +// are converted to the v1 piece CID plus padded/raw sizes used by parked_pieces. +func storachaPieceInfoFromFileName(name string) (*mk20.PieceInfo, bool, error) { + pcidStr, ok := strings.CutSuffix(name, ".car") + if !ok || pcidStr == "" { + return nil, false, nil + } + + pcid2, err := cid.Parse(pcidStr) + if err != nil { + return nil, false, nil + } + + pi, err := mk20.GetPieceInfo(pcid2) + if err != nil { + return nil, false, xerrors.Errorf("getting piece info: %w", err) + } + + return pi, true, nil +} + +// importStagedStorachaPiece finishes one target/storacha-staging/.car. +// It first creates or reuses parked_pieces/refs in a committed DB transaction. +// If an existing incomplete parked_piece still has a live task, it returns +// Imported=false and leaves the staged file untouched for a later retry. +// Otherwise it consumes the staged file: remove it if the final s-t00- +// already exists, or rename it to that final path. It then declares the piece +// location and marks parked_pieces.complete unless the row was already complete. +func importStagedStorachaPiece(ctx context.Context, dep *deps.Deps, storageID storiface.ID, targetPath, stagingPath string) (storachaImportResult, error) { + info, err := os.Stat(stagingPath) + if err != nil { + return storachaImportResult{}, xerrors.Errorf("checking staging file: %w", err) + } + if info.IsDir() { + return storachaImportResult{}, nil + } + + pi, ok, err := storachaPieceInfoFromFileName(filepath.Base(stagingPath)) + if err != nil { + return storachaImportResult{}, err + } + if !ok { + return storachaImportResult{}, nil + } + + var state storachaParkedPieceState + // The transaction only decides ownership and DB references. It deliberately + // does not move the file to s-t00-; the id must be committed before the + // filename depends on it. + comm, err := dep.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + parkedPieceID, createdParkedPiece, err := parkpiece.UpsertSkipWithInserted(tx, pi.PieceCIDV1.String(), int64(pi.Size), int64(pi.RawSize), true, true) + if err != nil { + return false, xerrors.Errorf("upsert parked piece: %w", err) + } + + state = storachaParkedPieceState{ + ID: parkedPieceID, + UseStagedFile: createdParkedPiece, + } + + if !createdParkedPiece { + state, err = claimExistingStorachaPieceTx(tx, parkedPieceID) + if err != nil { + return false, err + } + } + + if !state.UseStagedFile { + return true, nil + } + + if err := ensureStorachaRefsTx(tx, parkedPieceID, pi); err != nil { + return false, err + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return storachaImportResult{}, xerrors.Errorf("upsert parked piece: %w", err) + } + if !comm { + return storachaImportResult{}, xerrors.Errorf("failed to commit transaction") + } + if !state.UseStagedFile { + // An existing incomplete parked_piece still has a live harmony task. Leave + // the staged file in place; a later retry can use it if that task disappears. + return storachaImportResult{}, nil + } + + finalPath := storachaFinalPiecePath(targetPath, state.ID) + finalInfo, err := os.Stat(finalPath) + switch { + case err == nil: + // A final file means an earlier run got past rename. The staged file is + // now only a duplicate input for the same parked piece. + if finalInfo.IsDir() { + return storachaImportResult{}, xerrors.Errorf("final piece path is a directory: %s", finalPath) + } + if err := os.Remove(stagingPath); err != nil { + return storachaImportResult{}, xerrors.Errorf("removing staged duplicate: %w", err) + } + case os.IsNotExist(err): + // Normal post-commit path: the DB id is durable, so it is now safe for the + // final filename to depend on parked_pieces.id. + if err := os.Rename(stagingPath, finalPath); err != nil { + return storachaImportResult{}, xerrors.Errorf("renaming staging file: %w", err) + } + default: + return storachaImportResult{}, xerrors.Errorf("checking final piece file: %w", err) + } + + if err := declareStorachaPiece(ctx, dep, storageID, state.ID); err != nil { + return storachaImportResult{}, err + } + + if !state.AlreadyComplete { + // complete is last. If we crash before this, final-file recovery will see + // s-t00-, declare it again idempotently, and mark the row complete. + if err := markParkedPieceComplete(ctx, dep.DB, state.ID); err != nil { + return storachaImportResult{}, err + } + } + + return storachaImportResult{Imported: true, PieceCID: pi.PieceCIDV1.String()}, nil +} + +// recoverFinalStorachaPieces scans incomplete storacha-migration rows where the +// DB/ref side already exists. For each row it checks whether the file has +// already reached its final s-t00- path. If yes, it finishes declaration and +// complete=true. If no final file exists, it leaves the row alone; there is +// nothing safe to do without either the staged file or the final file. +func recoverFinalStorachaPieces(ctx context.Context, dep *deps.Deps, storageID storiface.ID, targetPath string, out *importPiecesOutput, batchSize int) error { + if out.Count >= batchSize { + return nil + } + + var rows []storachaFinalRecoveryRow + err := dep.DB.Select(ctx, &rows, ` + SELECT pp.id, pp.piece_cid, pp.piece_padded_size, pp.piece_raw_size + FROM parked_pieces pp + WHERE pp.complete = FALSE + AND pp.skip = TRUE + AND pp.cleanup_task_id IS NULL + AND EXISTS ( + SELECT 1 + FROM parked_piece_refs ppr + WHERE ppr.piece_id = pp.id + AND ppr.data_url = $1 + AND ppr.long_term = TRUE + ) + ORDER BY pp.id + LIMIT $2`, storachaMigrationDataURL, batchSize-out.Count) + if err != nil { + return xerrors.Errorf("query final storacha pieces: %w", err) + } + + for _, row := range rows { + if out.Count >= batchSize { + break + } + + imported, err := recoverFinalStorachaPiece(ctx, dep, storageID, targetPath, row) + if err != nil { + return err + } + if imported { + out.Pieces = append(out.Pieces, row.PieceCID) + out.Count++ + } + } + + return nil +} + +// recoverFinalStorachaPiece handles the post-rename crash window for one DB row. +// It requires target/s-t00- to exist, rebuilds the piece info from the DB +// row, reuses the same claim/ref checks as staged import, then declares the +// final file and marks the parked piece complete. It returns false when the +// final file is missing or a live task still owns the parked_piece. +func recoverFinalStorachaPiece(ctx context.Context, dep *deps.Deps, storageID storiface.ID, targetPath string, row storachaFinalRecoveryRow) (bool, error) { + finalPath := storachaFinalPiecePath(targetPath, row.ID) + info, err := os.Stat(finalPath) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, xerrors.Errorf("checking final piece file: %w", err) + } + if info.IsDir() { + return false, xerrors.Errorf("final piece path is a directory: %s", finalPath) + } + + pcid, err := cid.Parse(row.PieceCID) + if err != nil { + return false, xerrors.Errorf("parsing piece cid %s: %w", row.PieceCID, err) + } + pi := &mk20.PieceInfo{ + PieceCIDV1: pcid, + Size: abi.PaddedPieceSize(row.PaddedSize), + RawSize: uint64(row.RawSize), + } + + var state storachaParkedPieceState + comm, err := dep.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + state, err = claimExistingStorachaPieceTx(tx, row.ID) + if err != nil { + return false, err + } + if !state.UseStagedFile { + return true, nil + } + if err := ensureStorachaRefsTx(tx, row.ID, pi); err != nil { + return false, err + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("claim final storacha piece: %w", err) + } + if !comm { + return false, xerrors.Errorf("failed to commit transaction") + } + if !state.UseStagedFile { + return false, nil + } + + if err := declareStorachaPiece(ctx, dep, storageID, row.ID); err != nil { + return false, err + } + if !state.AlreadyComplete { + if err := markParkedPieceComplete(ctx, dep.DB, row.ID); err != nil { + return false, err + } + } + + return true, nil +} + +// claimExistingStorachaPieceTx locks and classifies an existing parked_pieces +// row. Complete rows return AlreadyComplete=true and allow this migration to +// consume the staged file. Incomplete rows with a live harmony task return +// UseStagedFile=false. Incomplete rows without a live task are claimed by +// clearing task_id and setting skip=true, then this migration can finish them. +func claimExistingStorachaPieceTx(tx *harmonydb.Tx, parkedPieceID int64) (storachaParkedPieceState, error) { + var complete bool + var taskID sql.NullInt64 + var taskExists bool + err := tx.QueryRow(` + SELECT pp.complete, pp.task_id, ht.id IS NOT NULL AS task_exists + FROM parked_pieces pp + LEFT JOIN harmony_task ht ON ht.id = pp.task_id + WHERE pp.id = $1 + FOR UPDATE OF pp`, parkedPieceID).Scan(&complete, &taskID, &taskExists) + if err != nil { + return storachaParkedPieceState{}, xerrors.Errorf("query parked piece: %w", err) + } + + state := storachaParkedPieceState{ + ID: parkedPieceID, + AlreadyComplete: complete, + } + if complete { + state.UseStagedFile = true + return state, nil + } + if taskID.Valid && taskExists { + return state, nil + } + + n, err := tx.Exec(` + UPDATE parked_pieces + SET task_id = NULL, skip = TRUE + WHERE id = $1 + AND complete = FALSE + AND (task_id IS NULL OR task_id = $2)`, parkedPieceID, taskID.Int64) + if err != nil { + return storachaParkedPieceState{}, xerrors.Errorf("claim parked piece: %w", err) + } + if n != 1 { + return storachaParkedPieceState{}, xerrors.Errorf("claim parked piece: expected 1, got %d", n) + } + + state.UseStagedFile = true + return state, nil +} + +// ensureStorachaRefsTx enforces the intended one-to-one shape for this import. +// It locks storacha-migration parked_piece_refs for this parked piece, fails if +// more than one exists, inserts the missing ref if needed, then does the same +// for pdp_piecerefs through the unique piece_ref relation. Existing PDP refs +// must already point at the same service and piece CID. +func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceInfo) error { + var refs []struct { + RefID int64 `db:"ref_id"` + } + err := tx.Select(&refs, ` + SELECT ref_id + FROM parked_piece_refs + WHERE piece_id = $1 + AND data_url = $2 + AND long_term = TRUE + ORDER BY ref_id + FOR UPDATE`, parkedPieceID, storachaMigrationDataURL) + if err != nil { + return xerrors.Errorf("query storacha parked piece refs: %w", err) + } + if len(refs) > 1 { + return xerrors.Errorf("storacha parked piece refs: expected at most 1 for parked piece %d, got %d", parkedPieceID, len(refs)) + } + + var pieceRef int64 + if len(refs) == 1 { + pieceRef = refs[0].RefID + } else { + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) + RETURNING ref_id`, parkedPieceID, storachaMigrationDataURL).Scan(&pieceRef) + if err != nil { + return xerrors.Errorf("insert storacha parked piece ref: %w", err) + } + } + + var pdpRefs []struct { + ID int64 `db:"id"` + Service string `db:"service"` + PieceCID string `db:"piece_cid"` + } + err = tx.Select(&pdpRefs, ` + SELECT id, service, piece_cid + FROM pdp_piecerefs + WHERE piece_ref = $1 + FOR UPDATE`, pieceRef) + if err != nil { + return xerrors.Errorf("query pdp piece refs: %w", err) + } + if len(pdpRefs) > 1 { + return xerrors.Errorf("pdp piece refs: expected at most 1 for parked piece ref %d, got %d", pieceRef, len(pdpRefs)) + } + if len(pdpRefs) == 1 { + if pdpRefs[0].Service != serviceName || pdpRefs[0].PieceCID != pi.PieceCIDV1.String() { + return xerrors.Errorf("pdp piece ref %d does not match storacha migration piece %s", pdpRefs[0].ID, pi.PieceCIDV1.String()) + } + return nil + } + + needsSaveCache := padreader.PaddedSize(pi.RawSize).Padded() >= minSaveCacheSize + n, err := tx.Exec(` + INSERT INTO pdp_piecerefs (service, piece_cid, piece_ref, created_at, needs_save_cache) + VALUES ($1, $2, $3, NOW(), $4)`, serviceName, pi.PieceCIDV1.String(), pieceRef, needsSaveCache) + if err != nil { + return xerrors.Errorf("insert pdp piece ref: %w", err) + } + if n != 1 { + return xerrors.Errorf("insert pdp piece ref: expected 1, got %d", n) + } + + return nil +} + +// storachaFinalPiecePath mirrors Curio's piece-park sector naming: parked piece +// id N is stored as miner-zero piece sector s-t00-N under the target piece dir. +func storachaFinalPiecePath(targetPath string, parkedPieceID int64) string { + pieceNumber := storiface.PieceNumber(parkedPieceID) + return filepath.Join(targetPath, storiface.SectorName(pieceNumber.Ref().ID)) +} + +// declareStorachaPiece records the final piece file in sector_location. The +// storage index upsert is idempotent, so recovery can call this again after a +// crash between rename and complete=true. +func declareStorachaPiece(ctx context.Context, dep *deps.Deps, storageID storiface.ID, parkedPieceID int64) error { + pieceNumber := storiface.PieceNumber(parkedPieceID) + err := dep.Si.StorageDeclareSector(ctx, storageID, pieceNumber.Ref().ID, storiface.FTPiece, true) + if err != nil { + return xerrors.Errorf("declaring storacha piece: %w", err) + } + return nil +} + +// markParkedPieceComplete is intentionally the last state transition. Once this +// is true, ParkPieceTask will not try to download/write the piece, so the final +// file and sector_location declaration must already be in place. +func markParkedPieceComplete(ctx context.Context, db *harmonydb.DB, parkedPieceID int64) error { + n, err := db.Exec(ctx, `UPDATE parked_pieces SET complete = TRUE WHERE id = $1`, parkedPieceID) + if err != nil { + return xerrors.Errorf("updating parked pieces: %w", err) + } + if n != 1 { + return xerrors.Errorf("updating parked pieces: expected 1, got %d", n) + } + return nil +} diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go new file mode 100644 index 000000000..e3c6db115 --- /dev/null +++ b/cmd/curio/storacha-migration_test.go @@ -0,0 +1,718 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + commcid "github.com/filecoin-project/go-fil-commcid" + + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" +) + +type storachaMigrationTestEnv struct { + ctx context.Context + db *harmonydb.DB + dep *deps.Deps + storageID storiface.ID + sourceDir string + targetDir string + stagingDir string +} + +type storachaPieceFixture struct { + fileName string + info *mk20.PieceInfo + data []byte +} + +type storachaPieceFixtures struct { + t *testing.T + seed uint64 +} + +type testParkedPiece struct { + id int64 + complete bool + skip bool + longTerm bool + taskID sql.NullInt64 +} + +type testPDPRef struct { + ID int64 `db:"id"` + Service string `db:"service"` + PieceCID string `db:"piece_cid"` + PieceRef int64 `db:"piece_ref"` +} + +func TestStorachaPieceInfoFromFileName(t *testing.T) { + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + + pi, ok, err := storachaPieceInfoFromFileName(fx.fileName) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, fx.info.PieceCIDV1.String(), pi.PieceCIDV1.String()) + require.Equal(t, fx.info.Size, pi.Size) + require.Equal(t, fx.info.RawSize, pi.RawSize) + + pi, ok, err = storachaPieceInfoFromFileName("not-a-car.txt") + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, pi) + + pi, ok, err = storachaPieceInfoFromFileName("not-a-cid.car") + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, pi) + + var commP [32]byte + pcidV1, err := commcid.DataCommitmentV1ToCID(commP[:]) + require.NoError(t, err) + pi, ok, err = storachaPieceInfoFromFileName(pcidV1.String() + ".car") + require.Error(t, err) + require.False(t, ok) + require.Nil(t, pi) +} + +func TestStorachaMigrationFreshImport(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + writeSourcePiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{fx.info.PieceCIDV1.String()}, out.Pieces) + + pp := parkedPieceByCID(t, env, fx) + require.True(t, pp.complete) + require.True(t, pp.skip) + require.True(t, pp.longTerm) + require.False(t, pp.taskID.Valid) + + assertMissing(t, filepath.Join(env.sourceDir, fx.fileName)) + assertMissing(t, filepath.Join(env.stagingDir, fx.fileName)) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pp.id), fx.data) + assertStorachaRefs(t, env, pp.id, 1) + assertPDPRefs(t, env, pp.id, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pp.id, 1) +} + +func TestStorachaMigrationExistingCompleteMovesStagedWhenFinalMissing(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, true, false, nil) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + require.False(t, pp.skip) + assertMissing(t, filepath.Join(env.stagingDir, fx.fileName)) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationExistingCompleteRemovesStagedDuplicateWhenFinalExists(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, true, false, nil) + finalBytes := []byte("already-final") + writeStagedPiece(t, env, fx) + writeFinalPiece(t, env, pieceID, finalBytes) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + assertMissing(t, filepath.Join(env.stagingDir, fx.fileName)) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), finalBytes) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationClaimsIncompleteParkedPieceWithNoTask(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + require.True(t, pp.skip) + require.False(t, pp.taskID.Valid) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationClaimsIncompleteParkedPieceWithStaleTask(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + staleTaskID := int64(999999) + pieceID := insertParkedPiece(t, env, fx, false, false, &staleTaskID) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + require.True(t, pp.skip) + require.False(t, pp.taskID.Valid) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationLeavesLiveTaskPieceAlone(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + taskID := insertHarmonyTask(t, env, "live-park-piece") + pieceID := insertParkedPiece(t, env, fx, false, false, &taskID) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 0, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + require.False(t, pp.skip) + require.True(t, pp.taskID.Valid) + require.Equal(t, taskID, pp.taskID.Int64) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertStorachaRefs(t, env, pieceID, 0) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationReusesExistingRefs(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + pdpID := insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + refs := storachaRefIDs(t, env, pieceID) + require.Equal(t, []int64{refID}, refs) + pdpRefs := pdpRefsForPiece(t, env, pieceID) + require.Len(t, pdpRefs, 1) + require.Equal(t, pdpID, pdpRefs[0].ID) +} + +func TestStorachaMigrationInsertsMissingPDPRefForExistingParkedRef(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []int64{refID}, storachaRefIDs(t, env, pieceID)) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) +} + +func TestStorachaMigrationDuplicateStorachaRefsErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + insertStorachaParkedRef(t, env, pieceID) + insertStorachaParkedRef(t, env, pieceID) + writeStagedPiece(t, env, fx) + + _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "expected at most 1") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationMismatchedPDPRefErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, "wrong-piece-cid") + writeStagedPiece(t, env, fx) + + _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "does not match storacha migration piece") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationRecoversFinalFile(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + writeFinalPiece(t, env, pieceID, fx.data) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationFinalRecoveryMissingFileNoops(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 0, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationFinalRecoveryDirectoryErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) + + _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "final piece path is a directory") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationFinalRecoveryLeavesLiveTaskPieceAlone(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + taskID := insertHarmonyTask(t, env, "live-final") + pieceID := insertParkedPiece(t, env, fx, false, true, &taskID) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + writeFinalPiece(t, env, pieceID, fx.data) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 0, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + require.True(t, pp.taskID.Valid) + require.Equal(t, taskID, pp.taskID.Int64) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationBatchProcessesStagingBeforeSource(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + staged := pieces.next() + source := pieces.next() + writeStagedPiece(t, env, staged) + writeSourcePiece(t, env, source) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{staged.info.PieceCIDV1.String()}, out.Pieces) + + stagedPP := parkedPieceByCID(t, env, staged) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, stagedPP.id), staged.data) + assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) + assertMissing(t, filepath.Join(env.stagingDir, staged.fileName)) +} + +func TestStorachaMigrationBatchProcessesFinalRecoveryBeforeSource(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + recovered := pieces.next() + source := pieces.next() + pieceID := insertParkedPiece(t, env, recovered, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, recovered.info.PieceCIDV1.String()) + writeFinalPiece(t, env, pieceID, recovered.data) + writeSourcePiece(t, env, source) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{recovered.info.PieceCIDV1.String()}, out.Pieces) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) +} + +func TestStorachaMigrationInvalidStagedFileDoesNotConsumeBatch(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + source := pieces.next() + invalidStagedPath := filepath.Join(env.stagingDir, "not-a-cid.car") + writeFile(t, invalidStagedPath, []byte("invalid staged input")) + writeSourcePiece(t, env, source) + + out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{source.info.PieceCIDV1.String()}, out.Pieces) + + pp := parkedPieceByCID(t, env, source) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pp.id), source.data) + assertFileBytes(t, invalidStagedPath, []byte("invalid staged input")) +} + +func TestStorachaMigrationMissingSourcePathErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + missingSource := filepath.Join(filepath.Dir(env.sourceDir), "missing-source") + + _, err := runImportPieces(env.ctx, env.dep, missingSource, env.targetDir, 20) + require.ErrorContains(t, err, "reading directory") +} + +func TestStorachaMigrationSourceImportErrorsWhenStagingPathExists(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + source := pieces.next() + writeSourcePiece(t, env, source) + require.NoError(t, os.MkdirAll(filepath.Join(env.stagingDir, source.fileName), 0755)) + + _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "staging file already exists") + assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) +} + +func TestStorachaMigrationStagedImportFinalDirectoryErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + writeStagedPiece(t, env, fx) + require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) + + _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "final piece path is a directory") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertSectorLocation(t, env, pieceID, 0) +} + +func setupStorachaMigrationTest(t *testing.T) storachaMigrationTestEnv { + t.Helper() + + ctx := context.Background() + db, err := harmonydb.NewFromConfigWithITestID(t) + require.NoError(t, err) + + root := t.TempDir() + sourceDir := filepath.Join(root, "source") + storageDir := filepath.Join(root, "storage") + targetDir := filepath.Join(storageDir, storiface.FTPiece.String()) + stagingDir := filepath.Join(targetDir, "storacha-staging") + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(targetDir, 0755)) + + storageID := storiface.ID(uuid.New().String()) + meta := &storiface.LocalStorageMeta{ + ID: storageID, + Weight: 10, + CanSeal: true, + CanStore: true, + } + mb, err := json.MarshalIndent(meta, "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(storageDir, paths.MetaFile), mb, 0644)) + + bls := &paths.BasicLocalStorage{PathToJSON: filepath.Join(root, "storage.json")} + index := paths.NewDBIndex(nil, db) + localStore, err := paths.NewLocal(ctx, bls, index, "") + require.NoError(t, err) + require.NoError(t, localStore.OpenPath(ctx, storageDir)) + + return storachaMigrationTestEnv{ + ctx: ctx, + db: db, + dep: &deps.Deps{DB: db, Si: index, LocalStore: localStore}, + storageID: storageID, + sourceDir: sourceDir, + targetDir: targetDir, + stagingDir: stagingDir, + } +} + +func newStorachaPieceFixtures(t *testing.T) *storachaPieceFixtures { + t.Helper() + return &storachaPieceFixtures{t: t} +} + +func (f *storachaPieceFixtures) next() storachaPieceFixture { + f.t.Helper() + f.seed++ + return newStorachaPieceFixture(f.t, f.seed) +} + +func newStorachaPieceFixture(t *testing.T, seed uint64) storachaPieceFixture { + t.Helper() + + rawSize := uint64(127) + var commP [32]byte + binary.BigEndian.PutUint64(commP[:8], seed) + binary.BigEndian.PutUint64(commP[8:16], rawSize) + pcidV2, err := commcid.DataCommitmentToPieceCidv2(commP[:], rawSize) + require.NoError(t, err) + + info, err := mk20.GetPieceInfo(pcidV2) + require.NoError(t, err) + + return storachaPieceFixture{ + fileName: pcidV2.String() + ".car", + info: info, + data: bytes.Repeat([]byte{byte(seed % 251)}, int(rawSize)), + } +} + +func writeSourcePiece(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture) { + t.Helper() + writeFile(t, filepath.Join(env.sourceDir, fx.fileName), fx.data) +} + +func writeStagedPiece(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture) { + t.Helper() + writeFile(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) +} + +func writeFinalPiece(t *testing.T, env storachaMigrationTestEnv, pieceID int64, data []byte) { + t.Helper() + writeFile(t, storachaFinalPiecePath(env.targetDir, pieceID), data) +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, data, 0644)) +} + +func assertFileBytes(t *testing.T, path string, expected []byte) { + t.Helper() + actual, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, expected, actual) +} + +func assertMissing(t *testing.T, path string) { + t.Helper() + _, err := os.Stat(path) + require.True(t, os.IsNotExist(err), "expected %s to be missing, got %v", path, err) +} + +func insertParkedPiece(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture, complete, skip bool, taskID *int64) int64 { + t.Helper() + + var pieceID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, complete, long_term, skip, task_id) + VALUES ($1, $2, $3, $4, TRUE, $5, $6) + RETURNING id + `, fx.info.PieceCIDV1.String(), int64(fx.info.Size), int64(fx.info.RawSize), complete, skip, taskID).Scan(&pieceID) + require.NoError(t, err) + return pieceID +} + +func insertStorachaParkedRef(t *testing.T, env storachaMigrationTestEnv, pieceID int64) int64 { + t.Helper() + + var refID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) + RETURNING ref_id + `, pieceID, storachaMigrationDataURL).Scan(&refID) + require.NoError(t, err) + return refID +} + +func insertPDPRef(t *testing.T, env storachaMigrationTestEnv, refID int64, service, pieceCID string) int64 { + t.Helper() + + var pdpID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO pdp_piecerefs (service, piece_cid, piece_ref, created_at, needs_save_cache) + VALUES ($1, $2, $3, NOW(), FALSE) + RETURNING id + `, service, pieceCID, refID).Scan(&pdpID) + require.NoError(t, err) + return pdpID +} + +func insertHarmonyTask(t *testing.T, env storachaMigrationTestEnv, name string) int64 { + t.Helper() + + var taskID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO harmony_task (name, added_by, posted_time) + VALUES ($1, 1, CURRENT_TIMESTAMP) + RETURNING id + `, name).Scan(&taskID) + require.NoError(t, err) + return taskID +} + +func parkedPieceByCID(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture) testParkedPiece { + t.Helper() + + var row testParkedPiece + err := env.db.QueryRow(env.ctx, ` + SELECT id, complete, skip, long_term, task_id + FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND long_term = TRUE + AND cleanup_task_id IS NULL + `, fx.info.PieceCIDV1.String(), int64(fx.info.Size)).Scan(&row.id, &row.complete, &row.skip, &row.longTerm, &row.taskID) + require.NoError(t, err) + return row +} + +func parkedPieceByID(t *testing.T, env storachaMigrationTestEnv, pieceID int64) testParkedPiece { + t.Helper() + + var row testParkedPiece + err := env.db.QueryRow(env.ctx, ` + SELECT id, complete, skip, long_term, task_id + FROM parked_pieces + WHERE id = $1 + `, pieceID).Scan(&row.id, &row.complete, &row.skip, &row.longTerm, &row.taskID) + require.NoError(t, err) + return row +} + +func storachaRefIDs(t *testing.T, env storachaMigrationTestEnv, pieceID int64) []int64 { + t.Helper() + + var rows []struct { + RefID int64 `db:"ref_id"` + } + err := env.db.Select(env.ctx, &rows, ` + SELECT ref_id + FROM parked_piece_refs + WHERE piece_id = $1 + AND data_url = $2 + AND long_term = TRUE + ORDER BY ref_id + `, pieceID, storachaMigrationDataURL) + require.NoError(t, err) + + out := make([]int64, 0, len(rows)) + for _, row := range rows { + out = append(out, row.RefID) + } + return out +} + +func pdpRefsForPiece(t *testing.T, env storachaMigrationTestEnv, pieceID int64) []testPDPRef { + t.Helper() + + var rows []testPDPRef + err := env.db.Select(env.ctx, &rows, ` + SELECT p.id, p.service, p.piece_cid, p.piece_ref + FROM pdp_piecerefs p + JOIN parked_piece_refs r ON r.ref_id = p.piece_ref + WHERE r.piece_id = $1 + ORDER BY p.id + `, pieceID) + require.NoError(t, err) + return rows +} + +func assertStorachaRefs(t *testing.T, env storachaMigrationTestEnv, pieceID int64, expected int) { + t.Helper() + require.Len(t, storachaRefIDs(t, env, pieceID), expected) +} + +func assertPDPRefs(t *testing.T, env storachaMigrationTestEnv, pieceID int64, pieceCID string, expected int) { + t.Helper() + refs := pdpRefsForPiece(t, env, pieceID) + require.Len(t, refs, expected) + for _, ref := range refs { + require.Equal(t, serviceName, ref.Service) + require.Equal(t, pieceCID, ref.PieceCID) + } +} + +func assertSectorLocation(t *testing.T, env storachaMigrationTestEnv, pieceID int64, expected int) { + t.Helper() + + var count int + err := env.db.QueryRow(env.ctx, ` + SELECT COUNT(*) + FROM sector_location + WHERE miner_id = 0 + AND sector_num = $1 + AND sector_filetype = $2 + AND storage_id = $3 + `, pieceID, int(storiface.FTPiece), string(env.storageID)).Scan(&count) + require.NoError(t, err) + require.Equal(t, expected, count) +} diff --git a/cmd/curio/toolbox.go b/cmd/curio/toolbox.go index d478a9f3d..5aa7ee096 100644 --- a/cmd/curio/toolbox.go +++ b/cmd/curio/toolbox.go @@ -40,6 +40,7 @@ var toolboxCmd = &cli.Command{ fixMsgCmd, downgradeCmd, fixBoostMigrationCmd, + importPiecesCmd, }, } diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index 52dfa0d63..8f174d0b2 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -1384,6 +1384,7 @@ COMMANDS: fix-msg Updated DB with message data missing from chain node downgrade Downgrade a cluster's database to a previous software version. fix-boost-migration Fix Boost migration + import-pieces Imports already existing pieces from storage to piece park system help, h Shows a list of commands or help for one command OPTIONS: @@ -1436,6 +1437,21 @@ OPTIONS: --help, -h show help ``` +### curio toolbox import-pieces +``` +NAME: + curio toolbox import-pieces - Imports already existing pieces from storage to piece park system + +USAGE: + curio toolbox import-pieces [command options] + +OPTIONS: + --source value path to store the pieces from + --target value path to piece storage directory in Curio's attached permanent storage + --batch-size value number of pieces to move to permanent storage (default: 20) + --help, -h show help +``` + ## curio batch ``` NAME: From 9cde1fa15d6fc61e499d8563bee794b9a0f42c03 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 29 May 2026 17:17:24 +0400 Subject: [PATCH 2/9] output piece_cid_v2 --- cmd/curio/storacha-migration.go | 34 +++++++++++++++++++++------- cmd/curio/storacha-migration_test.go | 11 +++++---- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go index dc6d999b4..f22a36776 100644 --- a/cmd/curio/storacha-migration.go +++ b/cmd/curio/storacha-migration.go @@ -13,10 +13,10 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/curio/cmd/curio/internal/translations" "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/parkpiece" @@ -34,8 +34,8 @@ type importPiecesOutput struct { } type storachaImportResult struct { - Imported bool - PieceCID string + Imported bool + PieceCIDV2 string } type storachaParkedPieceState struct { @@ -53,7 +53,7 @@ type storachaFinalRecoveryRow struct { var importPiecesCmd = &cli.Command{ Name: "import-pieces", - Usage: translations.T("Imports already existing pieces from storage to piece park system"), + Usage: "Imports already existing pieces from storage to piece park system", Flags: []cli.Flag{ &cli.StringFlag{ Name: "source", @@ -183,7 +183,7 @@ func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath return out, err } if result.Imported { - out.Pieces = append(out.Pieces, result.PieceCID) + out.Pieces = append(out.Pieces, result.PieceCIDV2) out.Count++ } } @@ -242,7 +242,7 @@ func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath return out, err } if result.Imported { - out.Pieces = append(out.Pieces, result.PieceCID) + out.Pieces = append(out.Pieces, result.PieceCIDV2) out.Count++ } } @@ -398,7 +398,7 @@ func importStagedStorachaPiece(ctx context.Context, dep *deps.Deps, storageID st } } - return storachaImportResult{Imported: true, PieceCID: pi.PieceCIDV1.String()}, nil + return storachaImportResult{Imported: true, PieceCIDV2: strings.TrimSuffix(filepath.Base(stagingPath), ".car")}, nil } // recoverFinalStorachaPieces scans incomplete storacha-migration rows where the @@ -441,7 +441,11 @@ func recoverFinalStorachaPieces(ctx context.Context, dep *deps.Deps, storageID s return err } if imported { - out.Pieces = append(out.Pieces, row.PieceCID) + pcidV2, err := storachaPieceCIDV2FromRow(row) + if err != nil { + return err + } + out.Pieces = append(out.Pieces, pcidV2) out.Count++ } } @@ -449,6 +453,20 @@ func recoverFinalStorachaPieces(ctx context.Context, dep *deps.Deps, storageID s return nil } +func storachaPieceCIDV2FromRow(row storachaFinalRecoveryRow) (string, error) { + pcidV1, err := cid.Parse(row.PieceCID) + if err != nil { + return "", xerrors.Errorf("parsing piece cid %s: %w", row.PieceCID, err) + } + + pcidV2, err := commcid.PieceCidV2FromV1(pcidV1, uint64(row.RawSize)) + if err != nil { + return "", xerrors.Errorf("getting piece cid v2: %w", err) + } + + return pcidV2.String(), nil +} + // recoverFinalStorachaPiece handles the post-rename crash window for one DB row. // It requires target/s-t00- to exist, rebuilds the piece info from the DB // row, reuses the same claim/ref checks as staged import, then declares the diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go index e3c6db115..4baa0fb4f 100644 --- a/cmd/curio/storacha-migration_test.go +++ b/cmd/curio/storacha-migration_test.go @@ -33,6 +33,7 @@ type storachaMigrationTestEnv struct { } type storachaPieceFixture struct { + cidV2 string fileName string info *mk20.PieceInfo data []byte @@ -97,7 +98,8 @@ func TestStorachaMigrationFreshImport(t *testing.T) { out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) - require.Equal(t, []string{fx.info.PieceCIDV1.String()}, out.Pieces) + require.Equal(t, []string{fx.cidV2}, out.Pieces) + require.NotEqual(t, []string{fx.info.PieceCIDV1.String()}, out.Pieces) pp := parkedPieceByCID(t, env, fx) require.True(t, pp.complete) @@ -375,7 +377,7 @@ func TestStorachaMigrationBatchProcessesStagingBeforeSource(t *testing.T) { out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) - require.Equal(t, []string{staged.info.PieceCIDV1.String()}, out.Pieces) + require.Equal(t, []string{staged.cidV2}, out.Pieces) stagedPP := parkedPieceByCID(t, env, staged) assertFileBytes(t, storachaFinalPiecePath(env.targetDir, stagedPP.id), staged.data) @@ -397,7 +399,7 @@ func TestStorachaMigrationBatchProcessesFinalRecoveryBeforeSource(t *testing.T) out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) - require.Equal(t, []string{recovered.info.PieceCIDV1.String()}, out.Pieces) + require.Equal(t, []string{recovered.cidV2}, out.Pieces) pp := parkedPieceByID(t, env, pieceID) require.True(t, pp.complete) @@ -415,7 +417,7 @@ func TestStorachaMigrationInvalidStagedFileDoesNotConsumeBatch(t *testing.T) { out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) - require.Equal(t, []string{source.info.PieceCIDV1.String()}, out.Pieces) + require.Equal(t, []string{source.cidV2}, out.Pieces) pp := parkedPieceByCID(t, env, source) assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pp.id), source.data) @@ -527,6 +529,7 @@ func newStorachaPieceFixture(t *testing.T, seed uint64) storachaPieceFixture { require.NoError(t, err) return storachaPieceFixture{ + cidV2: pcidV2.String(), fileName: pcidV2.String() + ".car", info: info, data: bytes.Repeat([]byte{byte(seed % 251)}, int(rawSize)), From 6326850a45d6c5069649cc4ced309211cd7d2c89 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 3 Jun 2026 23:25:44 +0400 Subject: [PATCH 3/9] use sectorIndex declare --- cmd/curio/storacha-migration.go | 118 +++++++++++++-------------- cmd/curio/storacha-migration_test.go | 53 ++++++------ 2 files changed, 83 insertions(+), 88 deletions(-) diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go index f22a36776..70836ad9d 100644 --- a/cmd/curio/storacha-migration.go +++ b/cmd/curio/storacha-migration.go @@ -3,6 +3,7 @@ package main import ( "context" "database/sql" + "encoding/json" "fmt" "os" "path/filepath" @@ -17,9 +18,11 @@ import ( "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/alertmanager/curioalerting" "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/parkpiece" + "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/mk20" ) @@ -61,7 +64,7 @@ var importPiecesCmd = &cli.Command{ }, &cli.StringFlag{ Name: "target", - Usage: "path to piece storage directory in Curio's attached permanent storage", + Usage: "path to storage directory in Curio's attached permanent storage", }, &cli.IntFlag{ Name: "batch-size", @@ -77,9 +80,9 @@ var importPiecesCmd = &cli.Command{ The source directory has Storacha CAR files named by piece CID v2. Curio's piece park stores the same bytes under a parked_pieces id, so the final - filename becomes s-t00-. That means the migration must - first learn or create the parked_pieces row before it knows the final - filename. + file under the target storage root becomes piece/s-t00-. + That means the migration must first learn or create the parked_pieces + row before it knows the final filename. Every fresh source file is moved to target/storacha-staging first. That staging move is the first retry boundary: once the file leaves source, a @@ -93,12 +96,13 @@ var importPiecesCmd = &cli.Command{ file alone because that task owns the normal download/write path. 4. Otherwise, ensure exactly one storacha-migration parked_piece_refs row and exactly one pdp_piecerefs row for that parked ref. - 5. After the transaction commits, move the staged file to s-t00-, - declare it in sector_location, then mark parked_pieces.complete=true. + 5. After the transaction commits, move the staged file to + target/piece/s-t00-, declare it in sector_location, then mark + parked_pieces.complete=true. recoverFinalStorachaPieces exists for the one point where staging can no - longer help: if a previous run already renamed the file to s-t00- and - then crashed before declaring the file or marking the parked piece + longer help: if a previous run already renamed the file into target/piece + and then crashed before declaring the file or marking the parked piece complete. At that point there is no staged filename left to process, so recovery starts from the DB row/ref and checks whether the final file is already present. @@ -122,12 +126,14 @@ var importPiecesCmd = &cli.Command{ return xerrors.Errorf("resolving target path: %w", err) } - dep, err := deps.GetDepsCLI(ctx, cctx) + db, err := deps.MakeDB(cctx) if err != nil { return err } - out, err := runImportPieces(ctx, dep, sourcePath, targetPath, cctx.Int("batch-size")) + si := paths.NewDBIndex(curioalerting.NewAlertingSystem(), db) + + out, err := runImportPieces(ctx, db, si, sourcePath, targetPath, cctx.Int("batch-size")) if err != nil { return err } @@ -141,15 +147,10 @@ var importPiecesCmd = &cli.Command{ }, } -func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { +func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { var out importPiecesOutput - targetPath, err := filepath.Abs(filepath.Clean(targetPath)) - if err != nil { - return out, xerrors.Errorf("resolving target path: %w", err) - } - - storageID, err := pieceStorageID(ctx, dep, targetPath) + storageID, err := pieceStorageID(targetPath) if err != nil { return out, err } @@ -178,7 +179,7 @@ func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath continue } - result, err := importStagedStorachaPiece(ctx, dep, storageID, targetPath, filepath.Join(staging, entry.Name())) + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, filepath.Join(staging, entry.Name())) if err != nil { return out, err } @@ -191,7 +192,7 @@ func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath // This covers the crash point after staging was renamed to the final // piece path, but before sector_location/parked_pieces was finalized. - err = recoverFinalStorachaPieces(ctx, dep, storageID, targetPath, &out, batchSize) + err = recoverFinalStorachaPieces(ctx, db, si, storageID, targetPath, &out, batchSize) if err != nil { return out, err } @@ -237,7 +238,7 @@ func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath return out, xerrors.Errorf("renaming old path: %w", err) } - result, err := importStagedStorachaPiece(ctx, dep, storageID, targetPath, stagingPath) + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath) if err != nil { return out, err } @@ -251,27 +252,24 @@ func runImportPieces(ctx context.Context, dep *deps.Deps, sourcePath, targetPath } // pieceStorageID maps the user-provided target directory back to Curio's -// storage id. The target flag is expected to point at an attached storage -// path's "piece" directory, because StorageDeclareSector needs the parent -// storage id when it writes sector_location. -func pieceStorageID(ctx context.Context, dep *deps.Deps, targetPath string) (storiface.ID, error) { - localPaths, err := dep.LocalStore.Local(ctx) +// storage id. The target flag is expected to point at the storage root that +// contains sectorstore.json; final piece files are stored under target/piece. +func pieceStorageID(targetPath string) (storiface.ID, error) { + mb, err := os.ReadFile(filepath.Join(targetPath, paths.MetaFile)) if err != nil { - return "", xerrors.Errorf("listing local storage paths: %w", err) + return "", xerrors.Errorf("reading storage metadata for %s: %w", targetPath, err) } - for _, localPath := range localPaths { - piecePath := filepath.Join(localPath.LocalPath, storiface.FTPiece.String()) - piecePath, err = filepath.Abs(filepath.Clean(piecePath)) - if err != nil { - return "", xerrors.Errorf("resolving piece path %s: %w", piecePath, err) - } - if piecePath == targetPath { - return localPath.ID, nil - } + var meta storiface.LocalStorageMeta + if err := json.Unmarshal(mb, &meta); err != nil { + return "", xerrors.Errorf("unmarshalling storage metadata for %s: %w", targetPath, err) + } + + if meta.ID != ("") { + return meta.ID, nil } - return "", xerrors.Errorf("target %s is not the piece directory of an attached local storage path", targetPath) + return "", xerrors.Errorf("no storage ID found for %s", targetPath) } // storachaPieceInfoFromFileName accepts only Storacha migration inputs named @@ -300,10 +298,11 @@ func storachaPieceInfoFromFileName(name string) (*mk20.PieceInfo, bool, error) { // It first creates or reuses parked_pieces/refs in a committed DB transaction. // If an existing incomplete parked_piece still has a live task, it returns // Imported=false and leaves the staged file untouched for a later retry. -// Otherwise it consumes the staged file: remove it if the final s-t00- -// already exists, or rename it to that final path. It then declares the piece -// location and marks parked_pieces.complete unless the row was already complete. -func importStagedStorachaPiece(ctx context.Context, dep *deps.Deps, storageID storiface.ID, targetPath, stagingPath string) (storachaImportResult, error) { +// Otherwise it consumes the staged file: remove it if the final +// target/piece/s-t00- already exists, or rename it to that final path. It +// then declares the piece location and marks parked_pieces.complete unless the +// row was already complete. +func importStagedStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, stagingPath string) (storachaImportResult, error) { info, err := os.Stat(stagingPath) if err != nil { return storachaImportResult{}, xerrors.Errorf("checking staging file: %w", err) @@ -324,7 +323,7 @@ func importStagedStorachaPiece(ctx context.Context, dep *deps.Deps, storageID st // The transaction only decides ownership and DB references. It deliberately // does not move the file to s-t00-; the id must be committed before the // filename depends on it. - comm, err := dep.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { parkedPieceID, createdParkedPiece, err := parkpiece.UpsertSkipWithInserted(tx, pi.PieceCIDV1.String(), int64(pi.Size), int64(pi.RawSize), true, true) if err != nil { return false, xerrors.Errorf("upsert parked piece: %w", err) @@ -386,14 +385,14 @@ func importStagedStorachaPiece(ctx context.Context, dep *deps.Deps, storageID st return storachaImportResult{}, xerrors.Errorf("checking final piece file: %w", err) } - if err := declareStorachaPiece(ctx, dep, storageID, state.ID); err != nil { + if err := declareStorachaPiece(ctx, si, storageID, state.ID); err != nil { return storachaImportResult{}, err } if !state.AlreadyComplete { // complete is last. If we crash before this, final-file recovery will see // s-t00-, declare it again idempotently, and mark the row complete. - if err := markParkedPieceComplete(ctx, dep.DB, state.ID); err != nil { + if err := markParkedPieceComplete(ctx, db, state.ID); err != nil { return storachaImportResult{}, err } } @@ -403,16 +402,17 @@ func importStagedStorachaPiece(ctx context.Context, dep *deps.Deps, storageID st // recoverFinalStorachaPieces scans incomplete storacha-migration rows where the // DB/ref side already exists. For each row it checks whether the file has -// already reached its final s-t00- path. If yes, it finishes declaration and -// complete=true. If no final file exists, it leaves the row alone; there is -// nothing safe to do without either the staged file or the final file. -func recoverFinalStorachaPieces(ctx context.Context, dep *deps.Deps, storageID storiface.ID, targetPath string, out *importPiecesOutput, batchSize int) error { +// already reached its final target/piece/s-t00- path. If yes, it finishes +// declaration and complete=true. If no final file exists, it leaves the row +// alone; there is nothing safe to do without either the staged file or the final +// file. +func recoverFinalStorachaPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath string, out *importPiecesOutput, batchSize int) error { if out.Count >= batchSize { return nil } var rows []storachaFinalRecoveryRow - err := dep.DB.Select(ctx, &rows, ` + err := db.Select(ctx, &rows, ` SELECT pp.id, pp.piece_cid, pp.piece_padded_size, pp.piece_raw_size FROM parked_pieces pp WHERE pp.complete = FALSE @@ -436,7 +436,7 @@ func recoverFinalStorachaPieces(ctx context.Context, dep *deps.Deps, storageID s break } - imported, err := recoverFinalStorachaPiece(ctx, dep, storageID, targetPath, row) + imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row) if err != nil { return err } @@ -468,11 +468,11 @@ func storachaPieceCIDV2FromRow(row storachaFinalRecoveryRow) (string, error) { } // recoverFinalStorachaPiece handles the post-rename crash window for one DB row. -// It requires target/s-t00- to exist, rebuilds the piece info from the DB -// row, reuses the same claim/ref checks as staged import, then declares the +// It requires target/piece/s-t00- to exist, rebuilds the piece info from the +// DB row, reuses the same claim/ref checks as staged import, then declares the // final file and marks the parked piece complete. It returns false when the // final file is missing or a live task still owns the parked_piece. -func recoverFinalStorachaPiece(ctx context.Context, dep *deps.Deps, storageID storiface.ID, targetPath string, row storachaFinalRecoveryRow) (bool, error) { +func recoverFinalStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath string, row storachaFinalRecoveryRow) (bool, error) { finalPath := storachaFinalPiecePath(targetPath, row.ID) info, err := os.Stat(finalPath) if os.IsNotExist(err) { @@ -496,7 +496,7 @@ func recoverFinalStorachaPiece(ctx context.Context, dep *deps.Deps, storageID st } var state storachaParkedPieceState - comm, err := dep.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { state, err = claimExistingStorachaPieceTx(tx, row.ID) if err != nil { return false, err @@ -519,11 +519,11 @@ func recoverFinalStorachaPiece(ctx context.Context, dep *deps.Deps, storageID st return false, nil } - if err := declareStorachaPiece(ctx, dep, storageID, row.ID); err != nil { + if err := declareStorachaPiece(ctx, si, storageID, row.ID); err != nil { return false, err } if !state.AlreadyComplete { - if err := markParkedPieceComplete(ctx, dep.DB, row.ID); err != nil { + if err := markParkedPieceComplete(ctx, db, row.ID); err != nil { return false, err } } @@ -654,18 +654,18 @@ func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceI } // storachaFinalPiecePath mirrors Curio's piece-park sector naming: parked piece -// id N is stored as miner-zero piece sector s-t00-N under the target piece dir. +// id N is stored as miner-zero piece sector s-t00-N under target/piece. func storachaFinalPiecePath(targetPath string, parkedPieceID int64) string { pieceNumber := storiface.PieceNumber(parkedPieceID) - return filepath.Join(targetPath, storiface.SectorName(pieceNumber.Ref().ID)) + return filepath.Join(targetPath, "piece", storiface.SectorName(pieceNumber.Ref().ID)) } // declareStorachaPiece records the final piece file in sector_location. The // storage index upsert is idempotent, so recovery can call this again after a // crash between rename and complete=true. -func declareStorachaPiece(ctx context.Context, dep *deps.Deps, storageID storiface.ID, parkedPieceID int64) error { +func declareStorachaPiece(ctx context.Context, si paths.SectorIndex, storageID storiface.ID, parkedPieceID int64) error { pieceNumber := storiface.PieceNumber(parkedPieceID) - err := dep.Si.StorageDeclareSector(ctx, storageID, pieceNumber.Ref().ID, storiface.FTPiece, true) + err := si.StorageDeclareSector(ctx, storageID, pieceNumber.Ref().ID, storiface.FTPiece, true) if err != nil { return xerrors.Errorf("declaring storacha piece: %w", err) } diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go index 4baa0fb4f..3768ed9fb 100644 --- a/cmd/curio/storacha-migration_test.go +++ b/cmd/curio/storacha-migration_test.go @@ -15,7 +15,6 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" - "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/storiface" @@ -25,7 +24,7 @@ import ( type storachaMigrationTestEnv struct { ctx context.Context db *harmonydb.DB - dep *deps.Deps + si paths.SectorIndex storageID storiface.ID sourceDir string targetDir string @@ -95,7 +94,7 @@ func TestStorachaMigrationFreshImport(t *testing.T) { fx := pieces.next() writeSourcePiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{fx.cidV2}, out.Pieces) @@ -122,7 +121,7 @@ func TestStorachaMigrationExistingCompleteMovesStagedWhenFinalMissing(t *testing pieceID := insertParkedPiece(t, env, fx, true, false, nil) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -145,7 +144,7 @@ func TestStorachaMigrationExistingCompleteRemovesStagedDuplicateWhenFinalExists( writeStagedPiece(t, env, fx) writeFinalPiece(t, env, pieceID, finalBytes) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -163,7 +162,7 @@ func TestStorachaMigrationClaimsIncompleteParkedPieceWithNoTask(t *testing.T) { pieceID := insertParkedPiece(t, env, fx, false, false, nil) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -185,7 +184,7 @@ func TestStorachaMigrationClaimsIncompleteParkedPieceWithStaleTask(t *testing.T) pieceID := insertParkedPiece(t, env, fx, false, false, &staleTaskID) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -205,7 +204,7 @@ func TestStorachaMigrationLeavesLiveTaskPieceAlone(t *testing.T) { pieceID := insertParkedPiece(t, env, fx, false, false, &taskID) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 0, out.Count) @@ -228,7 +227,7 @@ func TestStorachaMigrationReusesExistingRefs(t *testing.T) { pdpID := insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -247,7 +246,7 @@ func TestStorachaMigrationInsertsMissingPDPRefForExistingParkedRef(t *testing.T) refID := insertStorachaParkedRef(t, env, pieceID) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []int64{refID}, storachaRefIDs(t, env, pieceID)) @@ -263,7 +262,7 @@ func TestStorachaMigrationDuplicateStorachaRefsErrors(t *testing.T) { insertStorachaParkedRef(t, env, pieceID) writeStagedPiece(t, env, fx) - _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "expected at most 1") pp := parkedPieceByID(t, env, pieceID) @@ -281,7 +280,7 @@ func TestStorachaMigrationMismatchedPDPRefErrors(t *testing.T) { insertPDPRef(t, env, refID, serviceName, "wrong-piece-cid") writeStagedPiece(t, env, fx) - _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "does not match storacha migration piece") pp := parkedPieceByID(t, env, pieceID) @@ -299,7 +298,7 @@ func TestStorachaMigrationRecoversFinalFile(t *testing.T) { insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) writeFinalPiece(t, env, pieceID, fx.data) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -319,7 +318,7 @@ func TestStorachaMigrationFinalRecoveryMissingFileNoops(t *testing.T) { refID := insertStorachaParkedRef(t, env, pieceID) insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 0, out.Count) @@ -337,7 +336,7 @@ func TestStorachaMigrationFinalRecoveryDirectoryErrors(t *testing.T) { insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) - _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "final piece path is a directory") pp := parkedPieceByID(t, env, pieceID) @@ -355,7 +354,7 @@ func TestStorachaMigrationFinalRecoveryLeavesLiveTaskPieceAlone(t *testing.T) { insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) writeFinalPiece(t, env, pieceID, fx.data) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 0, out.Count) @@ -374,7 +373,7 @@ func TestStorachaMigrationBatchProcessesStagingBeforeSource(t *testing.T) { writeStagedPiece(t, env, staged) writeSourcePiece(t, env, source) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{staged.cidV2}, out.Pieces) @@ -396,7 +395,7 @@ func TestStorachaMigrationBatchProcessesFinalRecoveryBeforeSource(t *testing.T) writeFinalPiece(t, env, pieceID, recovered.data) writeSourcePiece(t, env, source) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{recovered.cidV2}, out.Pieces) @@ -414,7 +413,7 @@ func TestStorachaMigrationInvalidStagedFileDoesNotConsumeBatch(t *testing.T) { writeFile(t, invalidStagedPath, []byte("invalid staged input")) writeSourcePiece(t, env, source) - out, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 1) + out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{source.cidV2}, out.Pieces) @@ -428,7 +427,7 @@ func TestStorachaMigrationMissingSourcePathErrors(t *testing.T) { env := setupStorachaMigrationTest(t) missingSource := filepath.Join(filepath.Dir(env.sourceDir), "missing-source") - _, err := runImportPieces(env.ctx, env.dep, missingSource, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, missingSource, env.targetDir, 20) require.ErrorContains(t, err, "reading directory") } @@ -439,7 +438,7 @@ func TestStorachaMigrationSourceImportErrorsWhenStagingPathExists(t *testing.T) writeSourcePiece(t, env, source) require.NoError(t, os.MkdirAll(filepath.Join(env.stagingDir, source.fileName), 0755)) - _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "staging file already exists") assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) } @@ -452,7 +451,7 @@ func TestStorachaMigrationStagedImportFinalDirectoryErrors(t *testing.T) { writeStagedPiece(t, env, fx) require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) - _, err := runImportPieces(env.ctx, env.dep, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "final piece path is a directory") pp := parkedPieceByID(t, env, pieceID) @@ -471,10 +470,10 @@ func setupStorachaMigrationTest(t *testing.T) storachaMigrationTestEnv { root := t.TempDir() sourceDir := filepath.Join(root, "source") storageDir := filepath.Join(root, "storage") - targetDir := filepath.Join(storageDir, storiface.FTPiece.String()) + targetDir := storageDir stagingDir := filepath.Join(targetDir, "storacha-staging") require.NoError(t, os.MkdirAll(sourceDir, 0755)) - require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, storiface.FTPiece.String()), 0755)) storageID := storiface.ID(uuid.New().String()) meta := &storiface.LocalStorageMeta{ @@ -487,16 +486,12 @@ func setupStorachaMigrationTest(t *testing.T) storachaMigrationTestEnv { require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(storageDir, paths.MetaFile), mb, 0644)) - bls := &paths.BasicLocalStorage{PathToJSON: filepath.Join(root, "storage.json")} index := paths.NewDBIndex(nil, db) - localStore, err := paths.NewLocal(ctx, bls, index, "") - require.NoError(t, err) - require.NoError(t, localStore.OpenPath(ctx, storageDir)) return storachaMigrationTestEnv{ ctx: ctx, db: db, - dep: &deps.Deps{DB: db, Si: index, LocalStore: localStore}, + si: index, storageID: storageID, sourceDir: sourceDir, targetDir: targetDir, From 3d5b64cdae17085d0f159117f074ac8460a7f410 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 3 Jun 2026 23:38:51 +0400 Subject: [PATCH 4/9] check storage ID first --- cmd/curio/storacha-migration.go | 14 +++++----- cmd/curio/storacha-migration_test.go | 40 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go index 70836ad9d..8ca24123d 100644 --- a/cmd/curio/storacha-migration.go +++ b/cmd/curio/storacha-migration.go @@ -126,6 +126,11 @@ var importPiecesCmd = &cli.Command{ return xerrors.Errorf("resolving target path: %w", err) } + storageID, err := pieceStorageID(targetPath) + if err != nil { + return xerrors.Errorf("getting storage ID: %w", err) + } + db, err := deps.MakeDB(cctx) if err != nil { return err @@ -133,7 +138,7 @@ var importPiecesCmd = &cli.Command{ si := paths.NewDBIndex(curioalerting.NewAlertingSystem(), db) - out, err := runImportPieces(ctx, db, si, sourcePath, targetPath, cctx.Int("batch-size")) + out, err := runImportPieces(ctx, db, si, storageID, sourcePath, targetPath, cctx.Int("batch-size")) if err != nil { return err } @@ -147,14 +152,9 @@ var importPiecesCmd = &cli.Command{ }, } -func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { +func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { var out importPiecesOutput - storageID, err := pieceStorageID(targetPath) - if err != nil { - return out, err - } - staging := filepath.Join(targetPath, "storacha-staging") existing, err := os.ReadDir(staging) if err != nil { diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go index 3768ed9fb..9c5a17215 100644 --- a/cmd/curio/storacha-migration_test.go +++ b/cmd/curio/storacha-migration_test.go @@ -94,7 +94,7 @@ func TestStorachaMigrationFreshImport(t *testing.T) { fx := pieces.next() writeSourcePiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{fx.cidV2}, out.Pieces) @@ -121,7 +121,7 @@ func TestStorachaMigrationExistingCompleteMovesStagedWhenFinalMissing(t *testing pieceID := insertParkedPiece(t, env, fx, true, false, nil) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -144,7 +144,7 @@ func TestStorachaMigrationExistingCompleteRemovesStagedDuplicateWhenFinalExists( writeStagedPiece(t, env, fx) writeFinalPiece(t, env, pieceID, finalBytes) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -162,7 +162,7 @@ func TestStorachaMigrationClaimsIncompleteParkedPieceWithNoTask(t *testing.T) { pieceID := insertParkedPiece(t, env, fx, false, false, nil) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -184,7 +184,7 @@ func TestStorachaMigrationClaimsIncompleteParkedPieceWithStaleTask(t *testing.T) pieceID := insertParkedPiece(t, env, fx, false, false, &staleTaskID) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -204,7 +204,7 @@ func TestStorachaMigrationLeavesLiveTaskPieceAlone(t *testing.T) { pieceID := insertParkedPiece(t, env, fx, false, false, &taskID) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 0, out.Count) @@ -227,7 +227,7 @@ func TestStorachaMigrationReusesExistingRefs(t *testing.T) { pdpID := insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -246,7 +246,7 @@ func TestStorachaMigrationInsertsMissingPDPRefForExistingParkedRef(t *testing.T) refID := insertStorachaParkedRef(t, env, pieceID) writeStagedPiece(t, env, fx) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []int64{refID}, storachaRefIDs(t, env, pieceID)) @@ -262,7 +262,7 @@ func TestStorachaMigrationDuplicateStorachaRefsErrors(t *testing.T) { insertStorachaParkedRef(t, env, pieceID) writeStagedPiece(t, env, fx) - _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "expected at most 1") pp := parkedPieceByID(t, env, pieceID) @@ -280,7 +280,7 @@ func TestStorachaMigrationMismatchedPDPRefErrors(t *testing.T) { insertPDPRef(t, env, refID, serviceName, "wrong-piece-cid") writeStagedPiece(t, env, fx) - _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "does not match storacha migration piece") pp := parkedPieceByID(t, env, pieceID) @@ -298,7 +298,7 @@ func TestStorachaMigrationRecoversFinalFile(t *testing.T) { insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) writeFinalPiece(t, env, pieceID, fx.data) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 1, out.Count) @@ -318,7 +318,7 @@ func TestStorachaMigrationFinalRecoveryMissingFileNoops(t *testing.T) { refID := insertStorachaParkedRef(t, env, pieceID) insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 0, out.Count) @@ -336,7 +336,7 @@ func TestStorachaMigrationFinalRecoveryDirectoryErrors(t *testing.T) { insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) - _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "final piece path is a directory") pp := parkedPieceByID(t, env, pieceID) @@ -354,7 +354,7 @@ func TestStorachaMigrationFinalRecoveryLeavesLiveTaskPieceAlone(t *testing.T) { insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) writeFinalPiece(t, env, pieceID, fx.data) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.NoError(t, err) require.Equal(t, 0, out.Count) @@ -373,7 +373,7 @@ func TestStorachaMigrationBatchProcessesStagingBeforeSource(t *testing.T) { writeStagedPiece(t, env, staged) writeSourcePiece(t, env, source) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 1) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{staged.cidV2}, out.Pieces) @@ -395,7 +395,7 @@ func TestStorachaMigrationBatchProcessesFinalRecoveryBeforeSource(t *testing.T) writeFinalPiece(t, env, pieceID, recovered.data) writeSourcePiece(t, env, source) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 1) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{recovered.cidV2}, out.Pieces) @@ -413,7 +413,7 @@ func TestStorachaMigrationInvalidStagedFileDoesNotConsumeBatch(t *testing.T) { writeFile(t, invalidStagedPath, []byte("invalid staged input")) writeSourcePiece(t, env, source) - out, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 1) + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 1) require.NoError(t, err) require.Equal(t, 1, out.Count) require.Equal(t, []string{source.cidV2}, out.Pieces) @@ -427,7 +427,7 @@ func TestStorachaMigrationMissingSourcePathErrors(t *testing.T) { env := setupStorachaMigrationTest(t) missingSource := filepath.Join(filepath.Dir(env.sourceDir), "missing-source") - _, err := runImportPieces(env.ctx, env.db, env.si, missingSource, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, missingSource, env.targetDir, 20) require.ErrorContains(t, err, "reading directory") } @@ -438,7 +438,7 @@ func TestStorachaMigrationSourceImportErrorsWhenStagingPathExists(t *testing.T) writeSourcePiece(t, env, source) require.NoError(t, os.MkdirAll(filepath.Join(env.stagingDir, source.fileName), 0755)) - _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "staging file already exists") assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) } @@ -451,7 +451,7 @@ func TestStorachaMigrationStagedImportFinalDirectoryErrors(t *testing.T) { writeStagedPiece(t, env, fx) require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) - _, err := runImportPieces(env.ctx, env.db, env.si, env.sourceDir, env.targetDir, 20) + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) require.ErrorContains(t, err, "final piece path is a directory") pp := parkedPieceByID(t, env, pieceID) From 0003c5373a87814f012082919d3734dd2ad5dd5b Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 4 Jun 2026 17:46:51 +0400 Subject: [PATCH 5/9] always write in json file --- cmd/curio/storacha-migration.go | 52 ++++++++++++++++++++++++---- cmd/curio/storacha-migration_test.go | 18 ++++++++++ documentation/en/curio-cli/curio.md | 3 +- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go index 8ca24123d..a03c70cc5 100644 --- a/cmd/curio/storacha-migration.go +++ b/cmd/curio/storacha-migration.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -34,6 +35,7 @@ const storachaMigrationDataURL = "storacha-migration" type importPiecesOutput struct { Count int `json:"count"` Pieces []string `json:"pieces"` + Error string `json:"error,omitempty"` } type storachaImportResult struct { @@ -66,14 +68,19 @@ var importPiecesCmd = &cli.Command{ Name: "target", Usage: "path to storage directory in Curio's attached permanent storage", }, + &cli.StringFlag{ + Name: "result", + Usage: "path to write the JSON result", + }, &cli.IntFlag{ Name: "batch-size", Usage: "number of pieces to move to permanent storage", Value: 20, }, }, - Action: func(cctx *cli.Context) error { + Action: func(cctx *cli.Context) (err error) { ctx := cctx.Context + var out importPiecesOutput /* How this migration works: @@ -108,6 +115,29 @@ var importPiecesCmd = &cli.Command{ already present. */ + if cctx.String("result") == "" { + return fmt.Errorf("result is required") + } + + resultPath, err := homedir.Expand(cctx.String("result")) + if err != nil { + return xerrors.Errorf("expanding result path: %w", err) + } + resultPath, err = filepath.Abs(filepath.Clean(resultPath)) + if err != nil { + return xerrors.Errorf("resolving result path: %w", err) + } + + defer func() { + if err != nil { + out.Error = err.Error() + } + writeErr := writeImportPiecesResult(resultPath, out) + if writeErr != nil { + err = errors.Join(err, writeErr) + } + }() + if cctx.String("source") == "" || cctx.String("target") == "" { return fmt.Errorf("source and target both are required") } @@ -138,12 +168,7 @@ var importPiecesCmd = &cli.Command{ si := paths.NewDBIndex(curioalerting.NewAlertingSystem(), db) - out, err := runImportPieces(ctx, db, si, storageID, sourcePath, targetPath, cctx.Int("batch-size")) - if err != nil { - return err - } - - err = PrintJson(out) + out, err = runImportPieces(ctx, db, si, storageID, sourcePath, targetPath, cctx.Int("batch-size")) if err != nil { return err } @@ -152,6 +177,19 @@ var importPiecesCmd = &cli.Command{ }, } +func writeImportPiecesResult(path string, out importPiecesOutput) error { + resJson, err := json.MarshalIndent(out, "", " ") + if err != nil { + return xerrors.Errorf("marshalling import result: %w", err) + } + + if err := os.WriteFile(path, resJson, 0644); err != nil { + return xerrors.Errorf("writing import result: %w", err) + } + + return nil +} + func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { var out importPiecesOutput diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go index 9c5a17215..1fe9a1ca8 100644 --- a/cmd/curio/storacha-migration_test.go +++ b/cmd/curio/storacha-migration_test.go @@ -88,6 +88,24 @@ func TestStorachaPieceInfoFromFileName(t *testing.T) { require.Nil(t, pi) } +func TestWriteImportPiecesResultIncludesError(t *testing.T) { + path := filepath.Join(t.TempDir(), "result.json") + expected := importPiecesOutput{ + Count: 2, + Pieces: []string{"piece-a", "piece-b"}, + Error: "import failed", + } + + require.NoError(t, writeImportPiecesResult(path, expected)) + + mb, err := os.ReadFile(path) + require.NoError(t, err) + + var actual importPiecesOutput + require.NoError(t, json.Unmarshal(mb, &actual)) + require.Equal(t, expected, actual) +} + func TestStorachaMigrationFreshImport(t *testing.T) { env := setupStorachaMigrationTest(t) pieces := newStorachaPieceFixtures(t) diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index 8f174d0b2..28ef9fb94 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -1447,7 +1447,8 @@ USAGE: OPTIONS: --source value path to store the pieces from - --target value path to piece storage directory in Curio's attached permanent storage + --target value path to storage directory in Curio's attached permanent storage + --result value path to write the JSON result --batch-size value number of pieces to move to permanent storage (default: 20) --help, -h show help ``` From e4c20bacb70a6ba3132496c4df384f420c5c301e Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 9 Jun 2026 13:46:07 +0400 Subject: [PATCH 6/9] more parallel IPNI tasks --- tasks/indexing/task_pdp_v0_ipni.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/indexing/task_pdp_v0_ipni.go b/tasks/indexing/task_pdp_v0_ipni.go index 5e9a61993..faa915af7 100644 --- a/tasks/indexing/task_pdp_v0_ipni.go +++ b/tasks/indexing/task_pdp_v0_ipni.go @@ -317,10 +317,10 @@ func (P *PDPV0IPNITask) TypeDetails() harmonytask.TaskTypeDetails { Ram: 1 << 30, }, MaxFailures: 3, - IAmBored: passcall.Every(30*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { return P.schedule(context.Background(), taskFunc) }), - Max: P.max, + Max: taskhelp.Max(50), } } From b2c4fc6054e96d30b0ebc1c84b93777f6131cefe Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 22 Jul 2026 16:07:29 +0400 Subject: [PATCH 7/9] aggregate tool --- cmd/curio/storacha-migration-aggregate.go | 2510 +++++++++++++++++ .../storacha-migration-aggregate_test.go | 493 ++++ cmd/curio/toolbox.go | 1 + 3 files changed, 3004 insertions(+) create mode 100644 cmd/curio/storacha-migration-aggregate.go create mode 100644 cmd/curio/storacha-migration-aggregate_test.go diff --git a/cmd/curio/storacha-migration-aggregate.go b/cmd/curio/storacha-migration-aggregate.go new file mode 100644 index 000000000..712ca4aeb --- /dev/null +++ b/cmd/curio/storacha-migration-aggregate.go @@ -0,0 +1,2510 @@ +package main + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/ipfs/go-cid" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-data-segment/datasegment" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/alertmanager/curioalerting" + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" +) + +const storachaAggregateDefaultPieceSize = abi.PaddedPieceSize(1 << 30) + +// storachaAggregatePieceSize is a variable only so tests can exercise the +// recovery paths without writing 1 GiB aggregate files. Production uses the +// default 1 GiB padded aggregate size. +var storachaAggregatePieceSize = storachaAggregateDefaultPieceSize + +// aggregate-pieces is intentionally a file-backed pipeline, not a task table. +// +// The input is treated as read-only and is identified by its SHA256. That hash +// gives every distinct input file a distinct work directory, so changing the +// input never silently resumes with stale bucket/group data. +// +// The durable stages are: +// 1. raw buckets: stream the input, hydrate parked_piece ids from the DB, and +// append full records to padded-size bucket files with a cursor for resume. +// 2. deduped buckets: use the OS sort command per bucket, then drop adjacent +// duplicate PieceCIDv2 records. +// 3. groups.jsonl: write one atomic grouping file. This is the replay plan; a +// partial regroup would change aggregate membership and is not resumable. +// 4. complete/*.json: process each group into a real aggregate file and write +// a per-group commit marker after the piece is imported. +// +// Completed JSONL stage files are validated by size, record count, and SHA256 +// before they are trusted on retry. Aggregate body files use a different +// boundary: temp files are uncommitted scratch and can be overwritten, staged +// files are trusted only after staged/ exists, and final files are +// trusted from their DB row plus final path/size. After the staged marker exists +// we do not run CommP again; a rename or failed fsync can lose a directory entry +// but cannot change already-written bytes into another valid file. +type aggregatePiecesOutput struct { + Error string `json:"error,omitempty"` + Aggregates []aggregates `json:"aggregates"` +} + +// aggregates is the public result contract for this command. It reports only +// aggregates that reached a completion marker, so callers can safely consume a +// partial result even when the command exits with an error. +type aggregates struct { + PieceCid cid.Cid `json:"piece_cid"` + SubPieces []cid.Cid `json:"sub_pieces"` + Count int `json:"count"` +} + +// pieceDetails is the transient hydrated form of one input line. The JSONL +// bucket records below are the durable form; this type exists only while a DB +// hydration batch is in memory. +type pieceDetails struct { + PieceCidV2 cid.Cid + PieceCID cid.Cid `db:"piece_cid"` + PaddedSize abi.PaddedPieceSize `db:"piece_padded_size"` + RawSize uint64 + ID int64 `db:"id"` +} + +const ( + storachaAggregateManifestVersion = 1 + storachaAggregateHydrateBatch = 5000 + storachaAggregateGroupIDWidth = 20 + + storachaAggregateWorkDir = "storacha-aggregate-work" + storachaAggregateManifestFile = "manifest.json" + storachaAggregateBucketCursor = "bucket-cursor.json" + storachaAggregateRawBucketDir = "buckets.raw" + storachaAggregateDedupeDir = "buckets" + storachaAggregateGroupsFile = "groups.jsonl" + storachaAggregateCompleteDir = "complete" + storachaAggregateTempDir = "aggregate-tmp" + storachaAggregateStagingDirName = "storacha-aggregate-staging" + storachaAggregateStagedDirName = "staged" +) + +// storachaBucketRecord is the durable "full hydrated record" for one input +// piece. PieceCIDV2 is deliberately the first JSON field: OS sort compares +// whole JSONL lines, so putting the dedupe key first makes equal PieceCIDv2 +// records adjacent without a custom sorter or an in-memory seen set. +type storachaBucketRecord struct { + PieceCIDV2 string `json:"piece_cid_v2"` + PieceID int64 `json:"piece_id"` + PieceCIDV1 string `json:"piece_cid_v1"` + RawSize uint64 `json:"raw_size"` + PaddedSize uint64 `json:"padded_size"` +} + +// storachaAggregateGroup is the single atomic grouping artifact. Once this +// file exists, aggregate creation is just replaying deterministic work. That is +// important because a partial regrouping would change which subpieces belong to +// which aggregate and make resume semantics ambiguous. +type storachaAggregateGroup struct { + GroupID int64 `json:"group_id"` + PieceCIDV1 string `json:"piece_cid_v1"` + PaddedSize uint64 `json:"padded_size"` + Pieces []storachaBucketRecord `json:"pieces"` +} + +// storachaAggregateCompletion is a per-group commit marker. The aggregate file +// and DB rows may be recoverable after a crash, but this marker is the cheap +// local proof that this exact group was already imported and can be reported to +// the caller. +type storachaAggregateCompletion struct { + GroupID int64 `json:"group_id"` + PieceCID string `json:"piece_cid"` + SubPieces []string `json:"sub_pieces"` + Count int `json:"count"` + PieceCIDV1 string `json:"piece_cid_v1"` + RawSize uint64 `json:"raw_size"` + PaddedSize uint64 `json:"padded_size"` +} + +// storachaAggregateWorkManifest is the durable state machine for a single +// input hash. A stage is marked complete only after its files are fully written, +// synced, renamed where needed, and checksummed. +type storachaAggregateWorkManifest struct { + Version int `json:"version"` + InputSHA256 string `json:"input_sha256"` + InputSize int64 `json:"input_size"` + + RawBuckets storachaAggregateStageManifest `json:"raw_buckets"` + DedupedBuckets storachaAggregateStageManifest `json:"deduped_buckets"` + Groups storachaAggregateGroupManifest `json:"groups"` +} + +// storachaAggregateStageManifest describes append-only JSONL stage artifacts. +// It lets a retry trust a completed stage without rescanning all earlier input. +type storachaAggregateStageManifest struct { + Complete bool `json:"complete"` + Records int64 `json:"records"` + Files []storachaAggregateFileState `json:"files"` +} + +// storachaAggregateGroupManifest records the single grouping artifact. The +// piece count and group count are checked again before processing. +type storachaAggregateGroupManifest struct { + Complete bool `json:"complete"` + Pieces int64 `json:"pieces"` + Groups int64 `json:"groups"` + File storachaAggregateFileState `json:"file"` +} + +// storachaAggregateFileState is deliberately simple metadata: it catches torn +// JSONL records, external edits, and stale artifacts without needing a DB table. +type storachaAggregateFileState struct { + Name string `json:"name"` + Records int64 `json:"records"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +// storachaAggregateBucketCursorState is used only while raw bucketing is +// incomplete. The cursor is written after bucket files and required raw-bucket +// directory entries are synced, so on retry any bytes past these offsets are +// known-uncommitted tails and can be truncated. +type storachaAggregateBucketCursorState struct { + InputSHA256 string `json:"input_sha256"` + InputOffset int64 `json:"input_offset"` + InputLine int64 `json:"input_line"` + Records int64 `json:"records"` + BucketOffsets map[string]int64 `json:"bucket_offsets"` +} + +var aggregatePiecesCmd = &cli.Command{ + Name: "aggregate-pieces", + Usage: "Aggregates already existing pieces from storage into PODSI and park them", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "source", + Usage: "path to read the pieces from", + }, + &cli.StringFlag{ + Name: "target", + Usage: "path to storage directory in Curio's attached permanent storage", + }, + &cli.StringFlag{ + Name: "result", + Usage: "path to write the JSON result", + }, + &cli.StringFlag{ + Name: "input", + Usage: "path to read the CIDs from", + }, + }, + Action: func(cctx *cli.Context) (err error) { + ctx := cctx.Context + var workDir string + + if cctx.String("result") == "" { + return fmt.Errorf("result is required") + } + + resultPath, err := homedir.Expand(cctx.String("result")) + if err != nil { + return xerrors.Errorf("expanding result path: %w", err) + } + resultPath, err = filepath.Abs(filepath.Clean(resultPath)) + if err != nil { + return xerrors.Errorf("resolving result path: %w", err) + } + + defer func() { + // Always write the result file, including errors. This gives the + // caller the list of aggregates that are already committed and a + // machine-readable reason why the run stopped. + writeErr := writeAggregatePiecesResultFromWork(resultPath, workDir, err) + if writeErr != nil { + err = errors.Join(err, writeErr) + } + }() + + if cctx.String("source") == "" || cctx.String("target") == "" || cctx.String("input") == "" { + return fmt.Errorf("source, target and input are required") + } + + inputPath, err := homedir.Expand(cctx.String("input")) + if err != nil { + return xerrors.Errorf("expanding input path: %w", err) + } + inputPath, err = filepath.Abs(filepath.Clean(inputPath)) + if err != nil { + return xerrors.Errorf("expanding input path: %w", err) + } + + // Dedupe is intentionally delegated to the system sort implementation so + // the command can handle tens of millions of PieceCIDv2 records without + // building a process-wide seen set. + sortPath, err := exec.LookPath("sort") + if err != nil { + return xerrors.Errorf("aggregate-pieces requires OS sort in PATH: %w", err) + } + + sourcePath, err := homedir.Expand(cctx.String("source")) + if err != nil { + return xerrors.Errorf("expanding source path: %w", err) + } + sourcePath, err = filepath.Abs(filepath.Clean(sourcePath)) + if err != nil { + return xerrors.Errorf("resolving source path: %w", err) + } + + targetPath, err := homedir.Expand(cctx.String("target")) + if err != nil { + return xerrors.Errorf("expanding target path: %w", err) + } + targetPath, err = filepath.Abs(filepath.Clean(targetPath)) + if err != nil { + return xerrors.Errorf("resolving target path: %w", err) + } + + storageID, err := pieceStorageID(targetPath) + if err != nil { + return xerrors.Errorf("getting storage ID: %w", err) + } + + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + + si := paths.NewDBIndex(curioalerting.NewAlertingSystem(), db) + + workDir, err = runAggregatePieces(ctx, db, si, storageID, sortPath, inputPath, sourcePath, targetPath) + if err != nil { + return err + } + + return nil + }, +} + +func runAggregatePieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sortPath, inputPath, sourcePath, targetPath string) (string, error) { + // The input hash is part of the workdir path. If the user edits/replaces the + // input, the command starts a new deterministic pipeline instead of trying to + // reuse grouping state that was computed for different contents. + inputSHA256, inputSize, err := hashFileSHA256(inputPath) + if err != nil { + return "", xerrors.Errorf("hashing input file: %w", err) + } + + workDir := filepath.Join(targetPath, storachaAggregateWorkDir, inputSHA256) + if err := os.MkdirAll(workDir, 0755); err != nil { + return workDir, xerrors.Errorf("creating aggregate work directory: %w", err) + } + + manifest, err := loadStorachaAggregateManifest(workDir, inputSHA256, inputSize) + if err != nil { + return workDir, err + } + + // A completed stage is trusted only after its checksum/size/line-count still + // matches the manifest. If any completed artifact changed, the safest answer + // is to stop and require manual cleanup of this hash-specific workdir. + if err := validateCompletedStorachaAggregateStages(workDir, manifest); err != nil { + return workDir, err + } + + // The manifest is monotonic: raw buckets must complete before dedupe, and + // dedupe must complete before groups. The validation above checked the + // deepest completed stage we are resuming from. Each builder below only marks + // its stage complete after durable output is written and recorded, so the + // remaining control flow can stay as a simple forward pipeline. + if !manifest.RawBuckets.Complete { + if err := buildRawStorachaAggregateBuckets(ctx, db, inputPath, inputSHA256, workDir, manifest); err != nil { + return workDir, err + } + } + + if !manifest.DedupedBuckets.Complete { + if err := dedupeStorachaAggregateBuckets(ctx, sortPath, workDir, manifest); err != nil { + return workDir, err + } + } + + if !manifest.Groups.Complete { + if err := buildStorachaAggregateGroups(workDir, manifest); err != nil { + return workDir, err + } + } + + if err := processStorachaAggregateGroups(ctx, db, si, storageID, sourcePath, targetPath, workDir, manifest); err != nil { + return workDir, err + } + + return workDir, nil +} + +func hydrateStorachaSubPieces(ctx context.Context, db *harmonydb.DB, subPieces []pieceDetails) error { + for i := 0; i < len(subPieces); i += storachaAggregateHydrateBatch { + batchEnd := i + storachaAggregateHydrateBatch + if batchEnd > len(subPieces) { + batchEnd = len(subPieces) + } + + batch := subPieces[i:batchEnd] + pcids := make([]string, 0, len(batch)) + psizes := make([]int64, 0, len(batch)) + for _, piece := range batch { + pcids = append(pcids, piece.PieceCID.String()) + psizes = append(psizes, int64(piece.PaddedSize)) + } + + var dbPieces []struct { + Ord int64 `db:"ord"` + ID int64 `db:"id"` + } + // WITH ORDINALITY gives each input tuple a 1-based position inside this + // batch. That ordinal is not a DB identity; it is only the "slot number" + // of the element produced by unnest(). + // + // We need it because the input is allowed to contain duplicates. A plain + // join on piece CID/size would return the same parked_pieces row for each + // duplicate, but without the ordinal we could not safely write the id back + // to the matching slice element. The ordinal keeps hydration positional: + // input slot N gets the first matching complete parked piece for slot N. + err := db.Select(ctx, &dbPieces, ` + WITH input AS ( + SELECT u.piece_cid, u.piece_padded_size, u.ord + FROM unnest($1::text[], $2::bigint[]) WITH ORDINALITY AS u(piece_cid, piece_padded_size, ord) + ) + SELECT i.ord, pp.id + FROM input i + JOIN parked_pieces pp + ON pp.piece_cid = i.piece_cid + AND pp.piece_padded_size = i.piece_padded_size + AND pp.complete = TRUE + AND pp.long_term = TRUE + AND pp.cleanup_task_id IS NULL + ORDER BY i.ord, pp.id`, pcids, psizes) + if err != nil { + return xerrors.Errorf("querying parked_pieces: %w", err) + } + + for _, row := range dbPieces { + if row.Ord < 1 || int(row.Ord) > len(batch) { + return xerrors.Errorf("querying parked_pieces: invalid input ordinal %d", row.Ord) + } + idx := i + int(row.Ord) - 1 + if subPieces[idx].ID == 0 { + subPieces[idx].ID = row.ID + } + } + + for j := i; j < batchEnd; j++ { + if subPieces[j].ID == 0 { + return xerrors.Errorf("piece %s is not a complete active parked piece", subPieces[j].PieceCidV2) + } + } + } + + return nil +} + +func hashFileSHA256(path string) (string, int64, error) { + file, err := os.Open(path) + if err != nil { + return "", 0, xerrors.Errorf("opening %s: %w", path, err) + } + defer func() { + _ = file.Close() + }() + + h := sha256.New() + n, err := io.Copy(h, file) + if err != nil { + return "", 0, xerrors.Errorf("reading %s: %w", path, err) + } + + return hex.EncodeToString(h.Sum(nil)), n, nil +} + +func loadStorachaAggregateManifest(workDir, inputSHA256 string, inputSize int64) (*storachaAggregateWorkManifest, error) { + path := filepath.Join(workDir, storachaAggregateManifestFile) + mb, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return nil, xerrors.Errorf("reading aggregate manifest: %w", err) + } + + manifest := &storachaAggregateWorkManifest{ + Version: storachaAggregateManifestVersion, + InputSHA256: inputSHA256, + InputSize: inputSize, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return nil, err + } + return manifest, nil + } + + var manifest storachaAggregateWorkManifest + if err := json.Unmarshal(mb, &manifest); err != nil { + return nil, xerrors.Errorf("unmarshalling aggregate manifest: %w", err) + } + if manifest.Version != storachaAggregateManifestVersion { + return nil, xerrors.Errorf("aggregate manifest version mismatch: expected %d, got %d; cleanup %s manually before retrying", + storachaAggregateManifestVersion, manifest.Version, workDir) + } + // The workdir name already includes the SHA, but the manifest repeats both + // the SHA and byte length. If either differs, something outside this command + // moved or edited state and automatic recovery would be unsafe. + if manifest.InputSHA256 != inputSHA256 || manifest.InputSize != inputSize { + return nil, xerrors.Errorf("aggregate manifest input mismatch in %s; cleanup manually before retrying", workDir) + } + + return &manifest, nil +} + +func writeStorachaAggregateManifest(workDir string, manifest *storachaAggregateWorkManifest) error { + if err := writeJSONFileAtomic(filepath.Join(workDir, storachaAggregateManifestFile), manifest); err != nil { + return xerrors.Errorf("writing aggregate manifest: %w", err) + } + return nil +} + +func validateCompletedStorachaAggregateStages(workDir string, manifest *storachaAggregateWorkManifest) error { + if manifest.DedupedBuckets.Complete && !manifest.RawBuckets.Complete { + return xerrors.Errorf("aggregate manifest has dedupe complete before raw buckets; cleanup %s manually before retrying", workDir) + } + if manifest.Groups.Complete && !manifest.DedupedBuckets.Complete { + return xerrors.Errorf("aggregate manifest has groups complete before dedupe; cleanup %s manually before retrying", workDir) + } + + // Validate only the deepest completed stage. Each stage is derived entirely + // from the previous one, so once groups.jsonl is complete we only need to + // trust that replay plan; earlier buckets are no longer read. + switch { + case manifest.Groups.Complete: + return validateStorachaAggregateGroupFile(workDir, manifest) + case manifest.DedupedBuckets.Complete: + return validateAggregateFileStates(workDir, manifest.DedupedBuckets.Files) + case manifest.RawBuckets.Complete: + return validateAggregateFileStates(workDir, manifest.RawBuckets.Files) + default: + return nil + } +} + +func validateStorachaAggregateGroupFile(workDir string, manifest *storachaAggregateWorkManifest) error { + if !manifest.Groups.Complete { + return nil + } + + if err := validateAggregateFileState(workDir, manifest.Groups.File); err != nil { + return err + } + if manifest.Groups.File.Records != manifest.Groups.Groups { + return xerrors.Errorf("aggregate group manifest mismatch: file has %d groups, manifest has %d; cleanup %s manually before retrying", + manifest.Groups.File.Records, manifest.Groups.Groups, workDir) + } + + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + file, err := os.Open(groupsPath) + if err != nil { + return xerrors.Errorf("opening aggregate group file: %w", err) + } + defer func() { + _ = file.Close() + }() + + reader := bufio.NewReaderSize(file, 1024*1024) + var groups int64 + var pieces int64 + for { + line, ok, err := readJSONLLine(reader) + if err != nil { + return xerrors.Errorf("reading aggregate group file: %w", err) + } + if !ok { + break + } + + var group storachaAggregateGroup + if err := json.Unmarshal(line, &group); err != nil { + return xerrors.Errorf("decoding aggregate group %d: %w", groups, err) + } + if group.GroupID != groups { + return xerrors.Errorf("aggregate group id mismatch: expected %d, got %d; cleanup %s manually before retrying", groups, group.GroupID, workDir) + } + if err := validateStorachaAggregateGroup(group); err != nil { + return xerrors.Errorf("validating aggregate group %d: %w", group.GroupID, err) + } + + groups++ + pieces += int64(len(group.Pieces)) + } + + if groups != manifest.Groups.Groups { + return xerrors.Errorf("aggregate group count mismatch: file has %d, manifest has %d; cleanup %s manually before retrying", + groups, manifest.Groups.Groups, workDir) + } + if pieces != manifest.Groups.Pieces { + return xerrors.Errorf("aggregate group piece count mismatch: file has %d, manifest has %d; cleanup %s manually before retrying", + pieces, manifest.Groups.Pieces, workDir) + } + + return nil +} + +func validateAggregateFileStates(workDir string, states []storachaAggregateFileState) error { + for _, expected := range states { + if err := validateAggregateFileState(workDir, expected); err != nil { + return err + } + } + return nil +} + +func validateAggregateFileState(workDir string, expected storachaAggregateFileState) error { + actual, err := computeAggregateFileState(workDir, expected.Name) + if err != nil { + return err + } + if actual.Records != expected.Records || actual.Bytes != expected.Bytes || actual.SHA256 != expected.SHA256 { + return xerrors.Errorf("aggregate stage file %s changed; cleanup %s manually before retrying", expected.Name, workDir) + } + return nil +} + +func computeAggregateFileState(workDir, relName string) (storachaAggregateFileState, error) { + path := filepath.Join(workDir, filepath.FromSlash(relName)) + file, err := os.Open(path) + if err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("opening aggregate stage file %s: %w", relName, err) + } + defer func() { + _ = file.Close() + }() + + info, err := file.Stat() + if err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("checking aggregate stage file %s: %w", relName, err) + } + if info.IsDir() { + return storachaAggregateFileState{}, xerrors.Errorf("aggregate stage file is a directory: %s", relName) + } + + h := sha256.New() + reader := bufio.NewReaderSize(file, 1024*1024) + var records int64 + var bytesRead int64 + // Stage artifacts are JSONL and every committed record ends in '\n'. A file + // ending in a partial line is considered torn and is never recorded as a + // completed stage. + for { + line, ok, err := readJSONLLine(reader) + if err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("reading aggregate stage file %s: %w", relName, err) + } + if !ok { + break + } + if _, err := h.Write(line); err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("hashing aggregate stage file %s: %w", relName, err) + } + records++ + bytesRead += int64(len(line)) + } + + if bytesRead != info.Size() { + return storachaAggregateFileState{}, xerrors.Errorf("aggregate stage file %s changed while reading: stat=%d read=%d", relName, info.Size(), bytesRead) + } + + return storachaAggregateFileState{ + Name: filepath.ToSlash(relName), + Records: records, + Bytes: bytesRead, + SHA256: hex.EncodeToString(h.Sum(nil)), + }, nil +} + +func readJSONLLine(reader *bufio.Reader) ([]byte, bool, error) { + line, err := reader.ReadBytes('\n') + if err == nil { + return line, true, nil + } + if errors.Is(err, io.EOF) { + if len(line) == 0 { + return nil, false, nil + } + // EOF after bytes but before '\n' means a writer died mid-record. + // Callers either truncate back to a cursor or reject the completed stage. + return nil, false, xerrors.Errorf("torn JSONL record without trailing newline") + } + return nil, false, err +} + +func buildRawStorachaAggregateBuckets(ctx context.Context, db *harmonydb.DB, inputPath, inputSHA256, workDir string, manifest *storachaAggregateWorkManifest) error { + rawDir := filepath.Join(workDir, storachaAggregateRawBucketDir) + if err := os.MkdirAll(rawDir, 0755); err != nil { + return xerrors.Errorf("creating raw bucket directory: %w", err) + } + + // Raw bucketing is the only non-atomic stage because the input can be huge. + // The cursor gives us a two-file protocol: bucket files are fsynced first, + // newly referenced bucket directory entries are fsynced next, then the cursor + // is atomically replaced with the new input/bucket offsets. On retry we seek + // the input to the cursor and truncate buckets to the last committed offsets + // before appending more records. + cursor, err := loadStorachaAggregateBucketCursor(workDir, inputSHA256) + if err != nil { + return err + } + if err := truncateStorachaAggregateRawBuckets(rawDir, cursor); err != nil { + return err + } + + input, err := os.Open(inputPath) + if err != nil { + return xerrors.Errorf("opening aggregate input: %w", err) + } + defer func() { + _ = input.Close() + }() + + if _, err := input.Seek(cursor.InputOffset, io.SeekStart); err != nil { + return xerrors.Errorf("seeking aggregate input to cursor offset %d: %w", cursor.InputOffset, err) + } + + reader := bufio.NewReaderSize(input, 1024*1024) + batch := make([]pieceDetails, 0, storachaAggregateHydrateBatch) + offset := cursor.InputOffset + lineNumber := cursor.InputLine + records := cursor.Records + + flushBatch := func() error { + if len(batch) == 0 { + return nil + } + // Hydrate only a bounded batch into memory. The durable bucket record + // contains the parked_piece id, both CID forms, raw size, and padded size + // so later stages do not have to ask the DB again for subpiece metadata. + if err := hydrateStorachaSubPieces(ctx, db, batch); err != nil { + return err + } + if err := appendStorachaAggregateRawBucketBatch(rawDir, cursor, batch); err != nil { + return err + } + + // Cursor update is the commit point for this batch. If the process dies + // before this write, the next run truncates any bucket bytes from the + // failed attempt and re-reads these input lines. + records += int64(len(batch)) + cursor.InputOffset = offset + cursor.InputLine = lineNumber + cursor.Records = records + if err := writeStorachaAggregateBucketCursor(workDir, cursor); err != nil { + return err + } + + batch = batch[:0] + return nil + } + + for { + if err := ctx.Err(); err != nil { + return err + } + + line, readErr := reader.ReadBytes('\n') + if len(line) == 0 && errors.Is(readErr, io.EOF) { + break + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return xerrors.Errorf("reading aggregate input line %d: %w", lineNumber+1, readErr) + } + + offset += int64(len(line)) + lineNumber++ + + piece, err := parseStorachaAggregateInputLine(line, lineNumber) + if err != nil { + return err + } + batch = append(batch, piece) + + if len(batch) == storachaAggregateHydrateBatch { + if err := flushBatch(); err != nil { + return err + } + } + if errors.Is(readErr, io.EOF) { + break + } + } + + if err := flushBatch(); err != nil { + return err + } + if records == 0 { + return fmt.Errorf("input has no pieces") + } + + files, err := finishStorachaAggregateRawBuckets(workDir) + if err != nil { + return err + } + if len(files) == 0 { + return xerrors.Errorf("aggregate raw bucketing produced no files") + } + + manifest.RawBuckets = storachaAggregateStageManifest{ + Complete: true, + Records: records, + Files: files, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return err + } + + return nil +} + +func loadStorachaAggregateBucketCursor(workDir, inputSHA256 string) (*storachaAggregateBucketCursorState, error) { + path := filepath.Join(workDir, storachaAggregateBucketCursor) + mb, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return nil, xerrors.Errorf("reading aggregate bucket cursor: %w", err) + } + return &storachaAggregateBucketCursorState{ + InputSHA256: inputSHA256, + BucketOffsets: map[string]int64{}, + }, nil + } + + var cursor storachaAggregateBucketCursorState + if err := json.Unmarshal(mb, &cursor); err != nil { + return nil, xerrors.Errorf("unmarshalling aggregate bucket cursor: %w", err) + } + if cursor.InputSHA256 != inputSHA256 { + return nil, xerrors.Errorf("aggregate bucket cursor input mismatch in %s; cleanup manually before retrying", workDir) + } + if cursor.BucketOffsets == nil { + cursor.BucketOffsets = map[string]int64{} + } + return &cursor, nil +} + +func writeStorachaAggregateBucketCursor(workDir string, cursor *storachaAggregateBucketCursorState) error { + if err := writeJSONFileAtomic(filepath.Join(workDir, storachaAggregateBucketCursor), cursor); err != nil { + return xerrors.Errorf("writing aggregate bucket cursor: %w", err) + } + return nil +} + +func truncateStorachaAggregateRawBuckets(rawDir string, cursor *storachaAggregateBucketCursorState) error { + for _, size := range storachaAggregateBucketSizesAscending() { + name := storachaAggregateBucketFileName(size) + path := filepath.Join(rawDir, name) + offset := cursor.BucketOffsets[name] + + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + if offset != 0 { + return xerrors.Errorf("aggregate raw bucket %s is missing but cursor expects %d bytes; cleanup manually before retrying", path, offset) + } + continue + } + return xerrors.Errorf("checking aggregate raw bucket %s: %w", path, err) + } + if info.IsDir() { + return xerrors.Errorf("aggregate raw bucket is a directory: %s", path) + } + if offset > info.Size() { + return xerrors.Errorf("aggregate raw bucket %s is shorter than cursor offset %d; cleanup manually before retrying", path, offset) + } + if info.Size() != offset { + // The cursor is written only after bucket writes have been flushed and + // synced. Any bytes beyond the cursor are an uncommitted tail from a + // previous crash and must be rolled back before appending more JSONL. + if err := os.Truncate(path, offset); err != nil { + return xerrors.Errorf("truncating aggregate raw bucket %s to cursor offset %d: %w", path, offset, err) + } + } + } + + return nil +} + +func parseStorachaAggregateInputLine(line []byte, lineNumber int64) (pieceDetails, error) { + text := strings.TrimSpace(string(line)) + if text == "" { + return pieceDetails{}, xerrors.Errorf("aggregate input line %d is empty", lineNumber) + } + + pcidV2, err := cid.Parse(text) + if err != nil { + return pieceDetails{}, xerrors.Errorf("parsing aggregate input line %d: %w", lineNumber, err) + } + if !commcidv2.IsPieceCidV2(pcidV2) { + return pieceDetails{}, xerrors.Errorf("aggregate input line %d is not a PieceCIDv2: %s", lineNumber, pcidV2) + } + + // PieceCIDv2 carries the raw payload size. We read raw from the CID, then + // compute padded size from that raw value for parked_pieces lookup. We never + // derive raw size from padded/unpadded size in the aggregate path. + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return pieceDetails{}, xerrors.Errorf("decoding PieceCIDv2 on line %d: %w", lineNumber, err) + } + paddedSize := padreader.PaddedSize(rawSize).Padded() + if err := paddedSize.Validate(); err != nil { + return pieceDetails{}, xerrors.Errorf("invalid padded size for aggregate input line %d: %w", lineNumber, err) + } + if !storachaAggregateIsBucketSize(paddedSize) { + return pieceDetails{}, xerrors.Errorf("aggregate input line %d padded size %d is outside the supported 128-byte to %d-byte bucket range", lineNumber, paddedSize, storachaAggregatePieceSize) + } + + return pieceDetails{ + PieceCidV2: pcidV2, + PieceCID: pcidV1, + RawSize: rawSize, + PaddedSize: paddedSize, + }, nil +} + +type storachaAggregateRawBucketWriter struct { + file *os.File + writer *bufio.Writer + offset int64 + needsRawDirSync bool +} + +func appendStorachaAggregateRawBucketBatch(rawDir string, cursor *storachaAggregateBucketCursorState, batch []pieceDetails) error { + // One batch may touch multiple padded-size buckets. We open only the buckets + // needed for this batch, append JSONL, flush/fsync every touched file, and + // sync the raw bucket directory when a bucket becomes newly referenced. Only + // after those durability steps do we update cursor.BucketOffsets for the + // caller to commit. + writers := map[string]*storachaAggregateRawBucketWriter{} + closeWriters := func() error { + var err error + for _, writer := range writers { + if writer.file != nil { + err = errors.Join(err, writer.file.Close()) + } + } + return err + } + defer func() { + _ = closeWriters() + }() + + openWriter := func(name string) (*storachaAggregateRawBucketWriter, error) { + if writer, ok := writers[name]; ok { + return writer, nil + } + + path := filepath.Join(rawDir, name) + created := false + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0644) + if os.IsNotExist(err) { + file, err = os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY|os.O_APPEND, 0644) + if err == nil { + created = true + } + } + if err != nil { + return nil, xerrors.Errorf("opening aggregate raw bucket %s: %w", path, err) + } + + offset := cursor.BucketOffsets[name] + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, xerrors.Errorf("checking aggregate raw bucket %s: %w", path, err) + } + if info.Size() != offset { + _ = file.Close() + return nil, xerrors.Errorf("aggregate raw bucket %s has size %d, cursor expects %d; cleanup manually before retrying", path, info.Size(), offset) + } + + writer := &storachaAggregateRawBucketWriter{ + file: file, + writer: bufio.NewWriterSize(file, 1024*1024), + offset: offset, + needsRawDirSync: created || offset == 0, + } + writers[name] = writer + return writer, nil + } + + for _, piece := range batch { + record := storachaBucketRecord{ + PieceCIDV2: piece.PieceCidV2.String(), + PieceID: piece.ID, + PieceCIDV1: piece.PieceCID.String(), + RawSize: piece.RawSize, + PaddedSize: uint64(piece.PaddedSize), + } + + name := storachaAggregateBucketFileName(piece.PaddedSize) + writer, err := openWriter(name) + if err != nil { + return err + } + n, err := writeJSONLine(writer.writer, record) + if err != nil { + return xerrors.Errorf("writing aggregate raw bucket %s: %w", name, err) + } + writer.offset += int64(n) + } + + newOffsets := map[string]int64{} + needsRawDirSync := false + for name, writer := range writers { + if err := writer.writer.Flush(); err != nil { + return xerrors.Errorf("flushing aggregate raw bucket %s: %w", name, err) + } + if err := writer.file.Sync(); err != nil { + return xerrors.Errorf("syncing aggregate raw bucket %s: %w", name, err) + } + if err := writer.file.Close(); err != nil { + return xerrors.Errorf("closing aggregate raw bucket %s: %w", name, err) + } + writer.file = nil + newOffsets[name] = writer.offset + needsRawDirSync = needsRawDirSync || writer.needsRawDirSync + } + + if needsRawDirSync { + if err := syncDir(rawDir); err != nil { + return xerrors.Errorf("syncing aggregate raw bucket directory before cursor commit: %w", err) + } + } + + for name, offset := range newOffsets { + cursor.BucketOffsets[name] = offset + } + + return nil +} + +func finishStorachaAggregateRawBuckets(workDir string) ([]storachaAggregateFileState, error) { + rawDir := filepath.Join(workDir, storachaAggregateRawBucketDir) + var files []storachaAggregateFileState + var removedEmpty bool + + // Some size buckets may be opened but receive no records in small test runs + // or interrupted runs. Delete empties before recording the stage so later + // stages only scan files that contain data. + for _, size := range storachaAggregateBucketSizesAscending() { + rel := filepath.ToSlash(filepath.Join(storachaAggregateRawBucketDir, storachaAggregateBucketFileName(size))) + path := filepath.Join(workDir, filepath.FromSlash(rel)) + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, xerrors.Errorf("checking aggregate raw bucket %s: %w", path, err) + } + if info.IsDir() { + return nil, xerrors.Errorf("aggregate raw bucket is a directory: %s", path) + } + if info.Size() == 0 { + if err := os.Remove(path); err != nil { + return nil, xerrors.Errorf("removing empty aggregate raw bucket %s: %w", path, err) + } + removedEmpty = true + continue + } + + state, err := computeAggregateFileState(workDir, rel) + if err != nil { + return nil, err + } + files = append(files, state) + } + + if removedEmpty { + if err := syncDir(rawDir); err != nil { + return nil, err + } + } + + return files, nil +} + +func dedupeStorachaAggregateBuckets(ctx context.Context, sortPath, workDir string, manifest *storachaAggregateWorkManifest) error { + if !manifest.RawBuckets.Complete { + return xerrors.Errorf("aggregate raw buckets are not complete") + } + + dedupeDir := filepath.Join(workDir, storachaAggregateDedupeDir) + if err := os.MkdirAll(dedupeDir, 0755); err != nil { + return xerrors.Errorf("creating dedupe bucket directory: %w", err) + } + + // Dedupe is per padded-size bucket. Because PieceCIDv2 is the first JSON + // field, byte sorting the whole line groups duplicate input records together. + // This keeps memory bounded even when the input contains tens of millions of + // pieces. + var files []storachaAggregateFileState + var totalRecords int64 + for _, rawState := range manifest.RawBuckets.Files { + if err := ctx.Err(); err != nil { + return err + } + + rawPath := filepath.Join(workDir, filepath.FromSlash(rawState.Name)) + sortedTmp := filepath.Join(dedupeDir, filepath.Base(rawState.Name)+".sorted.tmp") + + cmd := exec.CommandContext(ctx, sortPath, "-o", sortedTmp, rawPath) + cmd.Env = append(os.Environ(), "LC_ALL=C") + sortOut, err := cmd.CombinedOutput() + if err != nil { + return xerrors.Errorf("sorting aggregate bucket %s: %w: %s", rawState.Name, err, strings.TrimSpace(string(sortOut))) + } + + finalRel := filepath.ToSlash(filepath.Join(storachaAggregateDedupeDir, filepath.Base(rawState.Name))) + finalPath := filepath.Join(workDir, filepath.FromSlash(finalRel)) + records, err := dedupeSortedStorachaAggregateBucket(sortedTmp, finalPath) + removeErr := os.Remove(sortedTmp) + if removeErr != nil && !os.IsNotExist(removeErr) { + err = errors.Join(err, xerrors.Errorf("removing sorted aggregate bucket temp %s: %w", sortedTmp, removeErr)) + } + if err != nil { + return err + } + if records == 0 { + if removeErr := os.Remove(finalPath); removeErr != nil && !os.IsNotExist(removeErr) { + return xerrors.Errorf("removing empty deduped aggregate bucket %s: %w", finalPath, removeErr) + } + continue + } + + state, err := computeAggregateFileState(workDir, finalRel) + if err != nil { + return err + } + if state.Records != records { + return xerrors.Errorf("deduped aggregate bucket %s record count changed while finalizing", finalRel) + } + files = append(files, state) + totalRecords += records + } + + if totalRecords == 0 { + return xerrors.Errorf("aggregate dedupe produced no records") + } + + manifest.DedupedBuckets = storachaAggregateStageManifest{ + Complete: true, + Records: totalRecords, + Files: files, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return err + } + + return nil +} + +func dedupeSortedStorachaAggregateBucket(sortedPath, finalPath string) (int64, error) { + input, err := os.Open(sortedPath) + if err != nil { + return 0, xerrors.Errorf("opening sorted aggregate bucket %s: %w", sortedPath, err) + } + defer func() { + _ = input.Close() + }() + + tmpPath := finalPath + ".tmp" + output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return 0, xerrors.Errorf("creating deduped aggregate bucket %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + reader := bufio.NewReaderSize(input, 1024*1024) + writer := bufio.NewWriterSize(output, 1024*1024) + var previousPieceCIDV2 string + var records int64 + // The sorted input makes duplicates adjacent. Keep the first hydrated record + // and drop later records for the same PieceCIDv2; all retained records are + // written back as canonical JSONL with a newline terminator. + for { + line, ok, err := readJSONLLine(reader) + if err != nil { + _ = output.Close() + return 0, xerrors.Errorf("reading sorted aggregate bucket %s: %w", sortedPath, err) + } + if !ok { + break + } + + var record storachaBucketRecord + if err := json.Unmarshal(line, &record); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("decoding sorted aggregate bucket %s: %w", sortedPath, err) + } + if err := validateStorachaBucketRecord(record); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("validating sorted aggregate bucket %s: %w", sortedPath, err) + } + if record.PieceCIDV2 == previousPieceCIDV2 { + continue + } + + if _, err := writeJSONLine(writer, record); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("writing deduped aggregate bucket %s: %w", tmpPath, err) + } + previousPieceCIDV2 = record.PieceCIDV2 + records++ + } + + if err := writer.Flush(); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("flushing deduped aggregate bucket %s: %w", tmpPath, err) + } + if err := output.Sync(); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("syncing deduped aggregate bucket %s: %w", tmpPath, err) + } + if err := output.Close(); err != nil { + return 0, xerrors.Errorf("closing deduped aggregate bucket %s: %w", tmpPath, err) + } + + if records == 0 { + removeTmp = false + if err := os.Remove(tmpPath); err != nil { + return 0, xerrors.Errorf("removing empty deduped aggregate bucket %s: %w", tmpPath, err) + } + return 0, nil + } + + if err := os.Rename(tmpPath, finalPath); err != nil { + return 0, xerrors.Errorf("renaming deduped aggregate bucket %s: %w", finalPath, err) + } + removeTmp = false + if err := syncDir(filepath.Dir(finalPath)); err != nil { + return 0, err + } + + return records, nil +} + +type storachaAggregateBucketReader struct { + // pending is the next unconsumed record from this size bucket. Keeping one + // record buffered per bucket lets grouping stream all buckets without loading + // bucket files into memory. + size abi.PaddedPieceSize + path string + file *os.File + reader *bufio.Reader + pending *storachaBucketRecord +} + +func buildStorachaAggregateGroups(workDir string, manifest *storachaAggregateWorkManifest) error { + if !manifest.DedupedBuckets.Complete { + return xerrors.Errorf("aggregate dedupe buckets are not complete") + } + + // Grouping reads from at most one file for each power-of-two padded size. + // For the 128-byte through 1-GiB range that is 24 readers, independent of + // how many total pieces are in the migration input. + readers, err := openStorachaAggregateBucketReaders(workDir, manifest.DedupedBuckets.Files) + if err != nil { + return err + } + defer func() { + for _, reader := range readers { + _ = reader.close() + } + }() + + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + tmpPath := groupsPath + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating aggregate group temp file: %w", err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + writer := bufio.NewWriterSize(file, 1024*1024) + maxEntries := datasegment.MaxIndexEntriesInDeal(storachaAggregatePieceSize) + var groupID int64 + var groupedPieces int64 + for groupedPieces < manifest.DedupedBuckets.Records { + placement := storachaAggregatePlacement{} + groupPieces := make([]storachaBucketRecord, 0, int(maxEntries)) + + // The greedy walk restarts from the largest bucket for every aggregate. + // Within a size bucket we keep sorted input order. We consume as many + // same-size pieces as fit, then move exactly one power smaller. With + // power-of-two piece sizes this keeps placement dense and deterministic. + for _, bucket := range readers { + for bucket.pending != nil { + nextOffsetNodes, fits, err := placement.tryAdd(abi.PaddedPieceSize(bucket.pending.PaddedSize), maxEntries) + if err != nil { + return xerrors.Errorf("checking aggregate fit for group %d: %w", groupID, err) + } + if !fits { + if len(groupPieces) == 0 { + return xerrors.Errorf("piece %s with padded size %d does not fit in a %d padded aggregate after reserving the datasegment index", + bucket.pending.PieceCIDV2, bucket.pending.PaddedSize, storachaAggregatePieceSize) + } + break + } + + groupPieces = append(groupPieces, *bucket.pending) + placement.accept(nextOffsetNodes) + if err := bucket.advance(); err != nil { + return err + } + } + } + + if len(groupPieces) == 0 { + return xerrors.Errorf("next aggregate subpiece does not fit in a %d padded aggregate after reserving the datasegment index", storachaAggregatePieceSize) + } + + // The fit check above is intentionally cheap. Before we persist the + // group, datasegment builds the aggregate metadata and computes the exact + // PieceCIDv1 that this replay plan must later produce. + expectedCIDV1, err := expectedStorachaAggregatePieceCID(groupPieces) + if err != nil { + return xerrors.Errorf("validating aggregate group %d with datasegment: %w", groupID, err) + } + + group := storachaAggregateGroup{ + GroupID: groupID, + PieceCIDV1: expectedCIDV1.String(), + PaddedSize: uint64(storachaAggregatePieceSize), + Pieces: groupPieces, + } + if _, err := writeJSONLine(writer, group); err != nil { + return xerrors.Errorf("writing aggregate group %d: %w", groupID, err) + } + + groupID++ + groupedPieces += int64(len(groupPieces)) + } + + if groupedPieces != manifest.DedupedBuckets.Records { + return xerrors.Errorf("aggregate grouping mismatch: grouped %d pieces, expected %d", groupedPieces, manifest.DedupedBuckets.Records) + } + + if err := writer.Flush(); err != nil { + _ = file.Close() + return xerrors.Errorf("flushing aggregate group file: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return xerrors.Errorf("syncing aggregate group file: %w", err) + } + if err := file.Close(); err != nil { + return xerrors.Errorf("closing aggregate group file: %w", err) + } + if err := os.Rename(tmpPath, groupsPath); err != nil { + return xerrors.Errorf("renaming aggregate group file: %w", err) + } + removeTmp = false + if err := syncDir(workDir); err != nil { + return err + } + + state, err := computeAggregateFileState(workDir, storachaAggregateGroupsFile) + if err != nil { + return err + } + manifest.Groups = storachaAggregateGroupManifest{ + Complete: true, + Pieces: groupedPieces, + Groups: groupID, + File: state, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return err + } + + return nil +} + +func openStorachaAggregateBucketReaders(workDir string, files []storachaAggregateFileState) ([]*storachaAggregateBucketReader, error) { + filesByBucket := map[string]storachaAggregateFileState{} + for _, state := range files { + filesByBucket[filepath.Base(state.Name)] = state + } + + var readers []*storachaAggregateBucketReader + // Readers are opened largest first so the grouping pass starts every + // aggregate with the largest available pieces, then walks one power smaller + // when the current size no longer fits. + for _, size := range storachaAggregateBucketSizesDescending() { + name := storachaAggregateBucketFileName(size) + state, ok := filesByBucket[name] + if !ok { + continue + } + + path := filepath.Join(workDir, filepath.FromSlash(state.Name)) + file, err := os.Open(path) + if err != nil { + for _, reader := range readers { + _ = reader.close() + } + return nil, xerrors.Errorf("opening deduped aggregate bucket %s: %w", state.Name, err) + } + + reader := &storachaAggregateBucketReader{ + size: size, + path: path, + file: file, + reader: bufio.NewReaderSize(file, 1024*1024), + } + if err := reader.advance(); err != nil { + _ = reader.close() + for _, existing := range readers { + _ = existing.close() + } + return nil, err + } + readers = append(readers, reader) + } + + return readers, nil +} + +func (r *storachaAggregateBucketReader) advance() error { + line, ok, err := readJSONLLine(r.reader) + if err != nil { + return xerrors.Errorf("reading deduped aggregate bucket %s: %w", r.path, err) + } + if !ok { + r.pending = nil + return nil + } + + var record storachaBucketRecord + if err := json.Unmarshal(line, &record); err != nil { + return xerrors.Errorf("decoding deduped aggregate bucket %s: %w", r.path, err) + } + if err := validateStorachaBucketRecord(record); err != nil { + return xerrors.Errorf("validating deduped aggregate bucket %s: %w", r.path, err) + } + if record.PaddedSize != uint64(r.size) { + return xerrors.Errorf("deduped aggregate bucket %s contains padded size %d, expected %d", r.path, record.PaddedSize, r.size) + } + + r.pending = &record + return nil +} + +func (r *storachaAggregateBucketReader) close() error { + if r.file == nil { + return nil + } + err := r.file.Close() + r.file = nil + return err +} + +type storachaAggregatePlacement struct { + nextOffsetNodes uint64 + count uint +} + +func (p storachaAggregatePlacement) tryAdd(size abi.PaddedPieceSize, maxEntries uint) (uint64, bool, error) { + if err := size.Validate(); err != nil { + return 0, false, err + } + if p.count >= maxEntries { + return 0, false, nil + } + + // This is the same alignment step used by datasegment.ComputeDealPlacement: + // choose the next location aligned for this piece size, then advance by one + // piece. We use it only to decide whether a candidate can fit cheaply while + // streaming bucket files. Before any group is committed, NewAggregate below + // recomputes placement and is the authoritative validation. + sizeInNodes := uint64(size) / 32 + index := (p.nextOffsetNodes + sizeInNodes - 1) / sizeInNodes + nextOffsetNodes := (index + 1) * sizeInNodes + packedPaddedSize := nextOffsetNodes * 32 + indexPaddedSize := uint64(maxEntries) * datasegment.EntrySize + + return nextOffsetNodes, packedPaddedSize+indexPaddedSize <= uint64(storachaAggregatePieceSize), nil +} + +func (p *storachaAggregatePlacement) accept(nextOffsetNodes uint64) { + p.nextOffsetNodes = nextOffsetNodes + p.count++ +} + +func validateStorachaAggregateGroup(group storachaAggregateGroup) error { + if group.PaddedSize != uint64(storachaAggregatePieceSize) { + return xerrors.Errorf("aggregate group padded size is %d, expected %d", group.PaddedSize, storachaAggregatePieceSize) + } + if len(group.Pieces) == 0 { + return xerrors.Errorf("aggregate group has no pieces") + } + for i, piece := range group.Pieces { + if err := validateStorachaBucketRecord(piece); err != nil { + return xerrors.Errorf("piece %d: %w", i, err) + } + } + + // Completed grouping state is trusted only if datasegment still agrees with + // the exact ordered list of subpieces and the precomputed aggregate CID. + expectedCIDV1, err := expectedStorachaAggregatePieceCID(group.Pieces) + if err != nil { + return err + } + if group.PieceCIDV1 != expectedCIDV1.String() { + return xerrors.Errorf("aggregate group commP mismatch: group has %s, datasegment computes %s", group.PieceCIDV1, expectedCIDV1) + } + + return nil +} + +func expectedStorachaAggregatePieceCID(records []storachaBucketRecord) (cid.Cid, error) { + // This computes the expected aggregate PieceCIDv1 from the subpiece CIDs and + // padded sizes only. The PieceCIDv2 for the aggregate is not known until the + // aggregate body is actually written, because v2 also carries raw bytes. + pinfos, err := storachaBucketRecordsToPieceInfos(records) + if err != nil { + return cid.Undef, err + } + aggr, err := datasegment.NewAggregate(storachaAggregatePieceSize, pinfos) + if err != nil { + return cid.Undef, err + } + pcid, err := aggr.PieceCID() + if err != nil { + return cid.Undef, xerrors.Errorf("computing aggregate PieceCIDv1: %w", err) + } + return pcid, nil +} + +func storachaBucketRecordsToPieceInfos(records []storachaBucketRecord) ([]abi.PieceInfo, error) { + pinfos := make([]abi.PieceInfo, 0, len(records)) + for i, record := range records { + pcidV1, err := cid.Parse(record.PieceCIDV1) + if err != nil { + return nil, xerrors.Errorf("parsing PieceCIDv1 for record %d: %w", i, err) + } + pinfos = append(pinfos, abi.PieceInfo{ + PieceCID: pcidV1, + Size: abi.PaddedPieceSize(record.PaddedSize), + }) + } + return pinfos, nil +} + +func validateStorachaBucketRecord(record storachaBucketRecord) error { + if record.PieceID <= 0 { + return xerrors.Errorf("invalid parked piece id %d", record.PieceID) + } + + // A bucket record is self-validating: PieceCIDv2 gives us PieceCIDv1 and raw + // size, raw size gives us padded size, and the bucket filename/checking code + // ensures the record lives in the expected power-of-two bucket. + pcidV2, err := cid.Parse(record.PieceCIDV2) + if err != nil { + return xerrors.Errorf("parsing PieceCIDv2 %q: %w", record.PieceCIDV2, err) + } + if !commcidv2.IsPieceCidV2(pcidV2) { + return xerrors.Errorf("not a PieceCIDv2: %s", pcidV2) + } + + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return xerrors.Errorf("decoding PieceCIDv2 %s: %w", pcidV2, err) + } + if pcidV1.String() != record.PieceCIDV1 { + return xerrors.Errorf("PieceCIDv1 mismatch for %s: record has %s, CID decodes to %s", pcidV2, record.PieceCIDV1, pcidV1) + } + if rawSize != record.RawSize { + return xerrors.Errorf("raw size mismatch for %s: record has %d, CID decodes to %d", pcidV2, record.RawSize, rawSize) + } + paddedSize := padreader.PaddedSize(rawSize).Padded() + if uint64(paddedSize) != record.PaddedSize { + return xerrors.Errorf("padded size mismatch for %s: record has %d, raw size pads to %d", pcidV2, record.PaddedSize, paddedSize) + } + if err := paddedSize.Validate(); err != nil { + return xerrors.Errorf("invalid padded size for %s: %w", pcidV2, err) + } + if !storachaAggregateIsBucketSize(paddedSize) { + return xerrors.Errorf("padded size %d is outside supported aggregate bucket range", paddedSize) + } + + return nil +} + +func processStorachaAggregateGroups(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath, workDir string, manifest *storachaAggregateWorkManifest) error { + for _, dir := range []string{ + filepath.Join(workDir, storachaAggregateCompleteDir), + filepath.Join(workDir, storachaAggregateTempDir), + storachaAggregateStagingDir(targetPath), + storachaAggregateStagedDir(targetPath), + } { + if err := os.MkdirAll(dir, 0755); err != nil { + return xerrors.Errorf("creating aggregate directory %s: %w", dir, err) + } + } + + // Processing is a linear replay of groups.jsonl. Group ids must match the + // file order so completion marker names are deterministic and partial result + // reporting can stream the same plan. + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + file, err := os.Open(groupsPath) + if err != nil { + return xerrors.Errorf("opening aggregate group file: %w", err) + } + defer func() { + _ = file.Close() + }() + + reader := bufio.NewReaderSize(file, 1024*1024) + var groups int64 + for { + if err := ctx.Err(); err != nil { + return err + } + + line, ok, err := readJSONLLine(reader) + if err != nil { + return xerrors.Errorf("reading aggregate group file: %w", err) + } + if !ok { + break + } + + var group storachaAggregateGroup + if err := json.Unmarshal(line, &group); err != nil { + return xerrors.Errorf("decoding aggregate group %d: %w", groups, err) + } + if group.GroupID != groups { + return xerrors.Errorf("aggregate group id mismatch while processing: expected %d, got %d", groups, group.GroupID) + } + if err := processStorachaAggregateGroup(ctx, db, si, storageID, sourcePath, targetPath, workDir, group); err != nil { + return err + } + groups++ + } + + if groups != manifest.Groups.Groups { + return xerrors.Errorf("processed %d aggregate groups, manifest expects %d; cleanup %s manually before retrying", groups, manifest.Groups.Groups, workDir) + } + + return nil +} + +func processStorachaAggregateGroup(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath, workDir string, group storachaAggregateGroup) error { + // Recovery order is from most committed to least committed: + // 1. completion marker: nothing else to do; + // 2. final Curio piece file + DB row: recreate metadata; + // 3. committed staging marker + staging file: run the normal import path; + // 4. no committed file: rebuild from read-only source pieces. + if _, ok, err := readCompletedStorachaAggregateGroup(workDir, group); err != nil { + return err + } else if ok { + return nil + } + + expectedCIDV1, err := cid.Parse(group.PieceCIDV1) + if err != nil { + return xerrors.Errorf("parsing aggregate group %d PieceCIDv1: %w", group.GroupID, err) + } + expectedPaddedSize := abi.PaddedPieceSize(group.PaddedSize) + + if recovered, err := recoverStorachaAggregateFromFinal(ctx, db, si, storageID, targetPath, workDir, group, expectedCIDV1, expectedPaddedSize); err != nil { + return err + } else if recovered { + return nil + } + + if recovered, err := recoverStorachaAggregateFromStaging(ctx, db, si, storageID, targetPath, workDir, group, expectedCIDV1, expectedPaddedSize); err != nil { + return err + } else if recovered { + return nil + } + + tempPath, verified, err := createStorachaAggregateTempFile(sourcePath, workDir, group, expectedCIDV1, expectedPaddedSize) + if err != nil { + return err + } + stagingPath, err := stageStorachaAggregateFile(targetPath, group, tempPath, verified) + if err != nil { + return err + } + return importStagedStorachaAggregateFile(ctx, db, si, storageID, targetPath, workDir, group, stagingPath, verified) +} + +func readCompletedStorachaAggregateGroup(workDir string, group storachaAggregateGroup) (storachaAggregateCompletion, bool, error) { + path := storachaAggregateCompletionPath(workDir, group.GroupID) + mb, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return storachaAggregateCompletion{}, false, nil + } + return storachaAggregateCompletion{}, false, xerrors.Errorf("reading aggregate completion marker %s: %w", path, err) + } + + var complete storachaAggregateCompletion + if err := json.Unmarshal(mb, &complete); err != nil { + return storachaAggregateCompletion{}, false, xerrors.Errorf("decoding aggregate completion marker %s: %w; cleanup manually before retrying", path, err) + } + // A marker is trusted only if it describes this exact group. A corrupt or + // stale marker would otherwise make us skip a group that still needs work. + if err := validateStorachaAggregateCompletion(group, complete); err != nil { + return storachaAggregateCompletion{}, false, xerrors.Errorf("invalid aggregate completion marker %s: %w; cleanup manually before retrying", path, err) + } + + return complete, true, nil +} + +func validateStorachaAggregateCompletion(group storachaAggregateGroup, complete storachaAggregateCompletion) error { + if complete.GroupID != group.GroupID { + return xerrors.Errorf("group id mismatch: marker has %d, group has %d", complete.GroupID, group.GroupID) + } + if complete.PieceCIDV1 != group.PieceCIDV1 { + return xerrors.Errorf("PieceCIDv1 mismatch: marker has %s, group has %s", complete.PieceCIDV1, group.PieceCIDV1) + } + if complete.PaddedSize != group.PaddedSize { + return xerrors.Errorf("padded size mismatch: marker has %d, group has %d", complete.PaddedSize, group.PaddedSize) + } + if complete.Count != len(group.Pieces) || len(complete.SubPieces) != len(group.Pieces) { + return xerrors.Errorf("subpiece count mismatch: marker has %d/%d, group has %d", complete.Count, len(complete.SubPieces), len(group.Pieces)) + } + pcidV2, err := cid.Parse(complete.PieceCID) + if err != nil { + return xerrors.Errorf("parsing aggregate PieceCIDv2 %q: %w", complete.PieceCID, err) + } + if !commcidv2.IsPieceCidV2(pcidV2) { + return xerrors.Errorf("aggregate PieceCID is not v2: %s", pcidV2) + } + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return xerrors.Errorf("decoding aggregate PieceCIDv2 %s: %w", pcidV2, err) + } + if pcidV1.String() != complete.PieceCIDV1 { + return xerrors.Errorf("aggregate PieceCIDv1 mismatch: marker has %s, v2 decodes to %s", complete.PieceCIDV1, pcidV1) + } + if rawSize != complete.RawSize { + return xerrors.Errorf("aggregate raw size mismatch: marker has %d, v2 decodes to %d", complete.RawSize, rawSize) + } + if uint64(padreader.PaddedSize(rawSize).Padded()) != complete.PaddedSize { + return xerrors.Errorf("aggregate padded size mismatch: marker has %d, raw size pads to %d", complete.PaddedSize, padreader.PaddedSize(rawSize).Padded()) + } + for i, piece := range group.Pieces { + if complete.SubPieces[i] != piece.PieceCIDV2 { + return xerrors.Errorf("subpiece %d mismatch: marker has %s, group has %s", i, complete.SubPieces[i], piece.PieceCIDV2) + } + } + return nil +} + +func recoverStorachaAggregateFromFinal(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, workDir string, group storachaAggregateGroup, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (bool, error) { + // A crash can happen after import-pieces moved the file into Curio storage + // but before aggregate-pieces wrote its completion marker. At this point the + // file has passed the staging marker boundary, so we trust the DB row and final + // filename instead of re-reading the whole aggregate for CommP. + var rows []storachaFinalRecoveryRow + err := db.Select(ctx, &rows, ` + SELECT id, piece_cid, piece_padded_size, piece_raw_size + FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND long_term = TRUE + AND cleanup_task_id IS NULL + ORDER BY id`, expectedCIDV1.String(), int64(expectedPaddedSize)) + if err != nil { + return false, xerrors.Errorf("querying aggregate parked pieces: %w", err) + } + + for _, row := range rows { + finalPath := storachaFinalPiecePath(targetPath, row.ID) + info, err := os.Stat(finalPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return false, xerrors.Errorf("checking aggregate final file %s: %w", finalPath, err) + } + if info.IsDir() { + return false, xerrors.Errorf("aggregate final path is a directory: %s", finalPath) + } + if info.Size() < 0 || uint64(info.Size()) != uint64(row.RawSize) { + return false, xerrors.Errorf("aggregate final file %s size mismatch for parked piece %d: db=%d file=%d; cleanup manually before retrying", finalPath, row.ID, row.RawSize, info.Size()) + } + + verified, err := storachaVerifiedAggregatePieceFromRow(row) + if err != nil { + return false, err + } + + imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row) + if err != nil { + return false, err + } + if !imported { + return false, nil + } + + if err := finishStorachaAggregateGroup(targetPath, workDir, group, verified); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +func recoverStorachaAggregateFromStaging(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, workDir string, group storachaAggregateGroup, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (bool, error) { + // Staged markers are the commit point for staging. The marker filename is the + // aggregate PieceCIDv2; if it decodes to this group, the matching .car has + // already been written, fsynced, named from CommP, and committed by marker. + staged := storachaAggregateStagedDir(targetPath) + entries, err := os.ReadDir(staged) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, xerrors.Errorf("reading aggregate staged marker directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + return false, xerrors.Errorf("aggregate staged marker is a directory: %s; cleanup manually before retrying", filepath.Join(staged, entry.Name())) + } + + pcidV2, err := cid.Parse(entry.Name()) + if err != nil || !commcidv2.IsPieceCidV2(pcidV2) { + return false, xerrors.Errorf("aggregate staged marker is not a PieceCIDv2: %s; cleanup manually before retrying", filepath.Join(staged, entry.Name())) + } + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return false, xerrors.Errorf("decoding aggregate staged marker %s: %w; cleanup manually before retrying", filepath.Join(staged, entry.Name()), err) + } + if !pcidV1.Equals(expectedCIDV1) || padreader.PaddedSize(rawSize).Padded() != expectedPaddedSize { + continue + } + + stagingPath := storachaAggregateStagingPath(targetPath, pcidV2) + info, err := os.Stat(stagingPath) + if err != nil { + if os.IsNotExist(err) { + return false, xerrors.Errorf("aggregate staged marker %s exists but staging file %s is missing; cleanup manually before retrying", filepath.Join(staged, entry.Name()), stagingPath) + } + return false, xerrors.Errorf("checking aggregate staging file %s: %w", stagingPath, err) + } + if info.IsDir() { + return false, xerrors.Errorf("aggregate staging path is a directory: %s", stagingPath) + } + if info.Size() < 0 || uint64(info.Size()) != rawSize { + return false, xerrors.Errorf("aggregate staging file %s size mismatch: marker raw=%d file=%d; cleanup manually before retrying", stagingPath, rawSize, info.Size()) + } + + verified := storachaVerifiedAggregatePiece{ + PieceCIDV2: pcidV2, + PieceCIDV1: pcidV1, + RawSize: rawSize, + PaddedSize: expectedPaddedSize, + } + + if err := importStagedStorachaAggregateFile(ctx, db, si, storageID, targetPath, workDir, group, stagingPath, verified); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +type storachaVerifiedAggregatePiece struct { + // RawSize is the exact number of bytes written/read for the aggregate file. + // PaddedSize and PieceCIDv1 come from CommP. PieceCIDv2 is computed from the + // same digest plus RawSize; there is no padded/unpadded-to-raw conversion. + PieceCIDV2 cid.Cid + PieceCIDV1 cid.Cid + RawSize uint64 + PaddedSize abi.PaddedPieceSize +} + +func storachaVerifiedAggregatePieceFromRow(row storachaFinalRecoveryRow) (storachaVerifiedAggregatePiece, error) { + pcidV1, err := cid.Parse(row.PieceCID) + if err != nil { + return storachaVerifiedAggregatePiece{}, xerrors.Errorf("parsing aggregate final PieceCIDv1 %s: %w", row.PieceCID, err) + } + + pcidV2, err := commcid.PieceCidV2FromV1(pcidV1, uint64(row.RawSize)) + if err != nil { + return storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate final PieceCIDv2 from DB row: %w", err) + } + + return storachaVerifiedAggregatePiece{ + PieceCIDV2: pcidV2, + PieceCIDV1: pcidV1, + RawSize: uint64(row.RawSize), + PaddedSize: abi.PaddedPieceSize(row.PaddedSize), + }, nil +} + +func createStorachaAggregateTempFile(sourcePath, workDir string, group storachaAggregateGroup, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (string, storachaVerifiedAggregatePiece, error) { + // Rebuild the datasegment aggregate from the deterministic group file, then + // stream read-only source/piece/s-t00- files into a local + // temp aggregate. The temp file is fsynced and handed to staging only after + // its CommP and padded size match the group plan. + pinfos, err := storachaBucketRecordsToPieceInfos(group.Pieces) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, err + } + + aggr, err := datasegment.NewAggregate(expectedPaddedSize, pinfos) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate group %d: %w", group.GroupID, err) + } + pcidV1, err := aggr.PieceCID() + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d PieceCIDv1: %w", group.GroupID, err) + } + if !pcidV1.Equals(expectedCIDV1) { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("aggregate group %d PieceCIDv1 changed: datasegment computes %s, group has %s", group.GroupID, pcidV1, expectedCIDV1) + } + + readers, closeReaders, err := openStorachaAggregateSubPieceReaders(sourcePath, group.Pieces) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, err + } + defer func() { + _ = closeReaders() + }() + + outR, err := aggr.AggregateObjectReader(readers) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate reader for group %d: %w", group.GroupID, err) + } + + tmpPath := storachaAggregateTempFilePath(workDir, group.GroupID) + if err := os.MkdirAll(filepath.Dir(tmpPath), 0755); err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate temp directory: %w", err) + } + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate temp file %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + cp := new(commp.Calc) + defer cp.Reset() + n, copyErr := io.Copy(io.MultiWriter(cp, file), outR) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(copyErr, syncErr, closeErr); err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("writing aggregate group %d: %w", group.GroupID, err) + } + + digest, actualPaddedSize, err := cp.Digest() + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d digest: %w", group.GroupID, err) + } + if abi.PaddedPieceSize(actualPaddedSize) != expectedPaddedSize { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("aggregate group %d padded size mismatch: expected %d, got %d", group.GroupID, expectedPaddedSize, actualPaddedSize) + } + + actualCIDV1, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d PieceCIDv1 from digest: %w", group.GroupID, err) + } + if !actualCIDV1.Equals(expectedCIDV1) { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("aggregate group %d commP mismatch: calculated %s, expected %s", group.GroupID, actualCIDV1, expectedCIDV1) + } + + // Raw size is the number of bytes AggregateObjectReader actually emitted. + // It is not inferred from padded size. This value is what PieceCIDv2 needs. + actualCIDV2, err := commcid.DataCommitmentToPieceCidv2(digest, uint64(n)) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d PieceCIDv2: %w", group.GroupID, err) + } + + removeTmp = false + + return tmpPath, storachaVerifiedAggregatePiece{ + PieceCIDV2: actualCIDV2, + PieceCIDV1: actualCIDV1, + RawSize: uint64(n), + PaddedSize: abi.PaddedPieceSize(actualPaddedSize), + }, nil +} + +func stageStorachaAggregateFile(targetPath string, group storachaAggregateGroup, tempPath string, verified storachaVerifiedAggregatePiece) (string, error) { + // This is the only place that turns a verified aggregate temp file into + // committed staging state. The order is: rename to .car, fsync + // directories, then write staged/. Staging recovery trusts only + // files that have crossed this marker boundary. + staging := storachaAggregateStagingDir(targetPath) + if err := os.MkdirAll(staging, 0755); err != nil { + return "", xerrors.Errorf("creating aggregate staging directory: %w", err) + } + if err := os.MkdirAll(storachaAggregateStagedDir(targetPath), 0755); err != nil { + return "", xerrors.Errorf("creating aggregate staged marker directory: %w", err) + } + + stagingPath := storachaAggregateStagingPath(targetPath, verified.PieceCIDV2) + if filepath.Clean(tempPath) == filepath.Clean(stagingPath) { + return "", xerrors.Errorf("aggregate group %d is already at staging path %s", group.GroupID, stagingPath) + } + + if info, err := os.Stat(stagingPath); err == nil { + if info.IsDir() { + return "", xerrors.Errorf("aggregate staging path is a directory: %s", stagingPath) + } + if info.Size() < 0 || uint64(info.Size()) != verified.RawSize { + return "", xerrors.Errorf("aggregate staging file %s size mismatch: expected %d, got %d; cleanup manually before retrying", stagingPath, verified.RawSize, info.Size()) + } + marked, err := storachaAggregateStagedMarkerExists(targetPath, verified.PieceCIDV2) + if err != nil { + return "", err + } + if marked { + return "", xerrors.Errorf("committed aggregate staging file already exists: %s; retry should recover it before staging another aggregate", stagingPath) + } + return "", xerrors.Errorf("uncommitted aggregate staging file already exists: %s; cleanup manually before retrying", stagingPath) + } else if !os.IsNotExist(err) { + return "", xerrors.Errorf("checking aggregate staging path %s: %w", stagingPath, err) + } + + if err := os.Rename(tempPath, stagingPath); err != nil { + return "", xerrors.Errorf("moving aggregate group %d to staging: %w", group.GroupID, err) + } + if err := syncDir(filepath.Dir(tempPath)); err != nil { + return "", err + } + if err := syncDir(staging); err != nil { + return "", err + } + if err := writeStorachaAggregateStagedMarker(targetPath, verified.PieceCIDV2); err != nil { + return "", err + } + + return stagingPath, nil +} + +func importStagedStorachaAggregateFile(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, workDir string, group storachaAggregateGroup, stagingPath string, verified storachaVerifiedAggregatePiece) error { + // This function assumes staging has already been committed by + // staged/. It does not create or refresh that marker, and it does + // not check it again or recompute CommP. It only performs cheap consistency + // checks, imports the staged file through the existing import-pieces path, then + // advances the group to complete/.json. + expectedStagingPath := storachaAggregateStagingPath(targetPath, verified.PieceCIDV2) + if filepath.Clean(stagingPath) != filepath.Clean(expectedStagingPath) { + return xerrors.Errorf("aggregate staging path mismatch: got %s, expected %s", stagingPath, expectedStagingPath) + } + + info, err := os.Stat(stagingPath) + if err != nil { + return xerrors.Errorf("checking aggregate staging file %s: %w", stagingPath, err) + } + if info.IsDir() { + return xerrors.Errorf("aggregate staging path is a directory: %s", stagingPath) + } + if info.Size() < 0 || uint64(info.Size()) != verified.RawSize { + return xerrors.Errorf("aggregate staging file %s size mismatch: expected %d, got %d; cleanup manually before retrying", stagingPath, verified.RawSize, info.Size()) + } + + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath) + if err != nil { + return err + } + if !result.Imported { + return xerrors.Errorf("aggregate piece %s is not importable yet", verified.PieceCIDV2) + } + + if err := finishStorachaAggregateGroup(targetPath, workDir, group, verified); err != nil { + return err + } + return nil +} + +func openStorachaAggregateSubPieceReaders(sourcePath string, subPieces []storachaBucketRecord) ([]io.Reader, func() error, error) { + // datasegment consumes one reader per subpiece in group order. We cap each + // reader at the raw size recorded from the PieceCIDv2 so stray bytes in a + // source file cannot affect the aggregate stream. + readers := make([]io.Reader, 0, len(subPieces)) + files := make([]*os.File, 0, len(subPieces)) + + closeReaders := func() error { + var err error + for _, file := range files { + err = errors.Join(err, file.Close()) + } + return err + } + + for _, piece := range subPieces { + // The source is read-only and is addressed exactly like Curio piece park: + // source/piece/s-t00-. We do not fall back to CID-named + // files here because aggregate-pieces is replaying already imported + // parked pieces, not consuming import-pieces source files. + piecePath := storachaFinalPiecePath(sourcePath, piece.PieceID) + file, err := os.Open(piecePath) + if err != nil { + _ = closeReaders() + return nil, nil, xerrors.Errorf("opening subpiece %s: %w", piecePath, err) + } + files = append(files, file) + + info, err := file.Stat() + if err != nil { + _ = closeReaders() + return nil, nil, xerrors.Errorf("checking subpiece %s: %w", piecePath, err) + } + if info.IsDir() { + _ = closeReaders() + return nil, nil, xerrors.Errorf("subpiece path is a directory: %s", piecePath) + } + if info.Size() < 0 || uint64(info.Size()) != piece.RawSize { + _ = closeReaders() + return nil, nil, xerrors.Errorf("subpiece %s has size %d, expected %d", piecePath, info.Size(), piece.RawSize) + } + + readers = append(readers, io.LimitReader(file, int64(piece.RawSize))) + } + + return readers, closeReaders, nil +} + +func writeStorachaAggregateCompletion(workDir string, group storachaAggregateGroup, verified storachaVerifiedAggregatePiece) error { + // This marker is written last. After it exists, retries skip the group and + // result generation reports the aggregate to the caller. + subPieces := make([]string, 0, len(group.Pieces)) + for _, piece := range group.Pieces { + subPieces = append(subPieces, piece.PieceCIDV2) + } + + complete := storachaAggregateCompletion{ + GroupID: group.GroupID, + PieceCID: verified.PieceCIDV2.String(), + SubPieces: subPieces, + Count: len(subPieces), + PieceCIDV1: group.PieceCIDV1, + RawSize: verified.RawSize, + PaddedSize: group.PaddedSize, + } + if err := writeJSONFileAtomic(storachaAggregateCompletionPath(workDir, group.GroupID), complete); err != nil { + return xerrors.Errorf("writing aggregate completion marker for group %d: %w", group.GroupID, err) + } + return nil +} + +func finishStorachaAggregateGroup(targetPath, workDir string, group storachaAggregateGroup, verified storachaVerifiedAggregatePiece) error { + if err := removeStorachaAggregateStagedMarker(targetPath, verified.PieceCIDV2); err != nil { + return err + } + return writeStorachaAggregateCompletion(workDir, group, verified) +} + +func writeStorachaAggregateStagedMarker(targetPath string, pcidV2 cid.Cid) error { + markerPath := storachaAggregateStagedMarkerPath(targetPath, pcidV2) + if err := os.MkdirAll(filepath.Dir(markerPath), 0755); err != nil { + return xerrors.Errorf("creating aggregate staged marker directory: %w", err) + } + + tmpPath := markerPath + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating aggregate staged marker temp %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(syncErr, closeErr); err != nil { + return xerrors.Errorf("writing aggregate staged marker temp %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, markerPath); err != nil { + return xerrors.Errorf("renaming aggregate staged marker %s: %w", markerPath, err) + } + removeTmp = false + if err := syncDir(filepath.Dir(markerPath)); err != nil { + return err + } + return nil +} + +func storachaAggregateStagedMarkerExists(targetPath string, pcidV2 cid.Cid) (bool, error) { + markerPath := storachaAggregateStagedMarkerPath(targetPath, pcidV2) + info, err := os.Stat(markerPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, xerrors.Errorf("checking aggregate staged marker %s: %w", markerPath, err) + } + if info.IsDir() { + return false, xerrors.Errorf("aggregate staged marker is a directory: %s; cleanup manually before retrying", markerPath) + } + return true, nil +} + +func removeStorachaAggregateStagedMarker(targetPath string, pcidV2 cid.Cid) error { + markerPath := storachaAggregateStagedMarkerPath(targetPath, pcidV2) + if err := os.Remove(markerPath); err != nil { + if os.IsNotExist(err) { + return nil + } + return xerrors.Errorf("removing aggregate staged marker %s: %w", markerPath, err) + } + if err := syncDir(filepath.Dir(markerPath)); err != nil { + return err + } + return nil +} + +func storachaAggregateStagingDir(targetPath string) string { + return filepath.Join(targetPath, storachaAggregateStagingDirName) +} + +func storachaAggregateStagedDir(targetPath string) string { + return filepath.Join(storachaAggregateStagingDir(targetPath), storachaAggregateStagedDirName) +} + +func storachaAggregateStagingPath(targetPath string, pcidV2 cid.Cid) string { + return filepath.Join(storachaAggregateStagingDir(targetPath), pcidV2.String()+".car") +} + +func storachaAggregateStagedMarkerPath(targetPath string, pcidV2 cid.Cid) string { + return filepath.Join(storachaAggregateStagedDir(targetPath), pcidV2.String()) +} + +func storachaAggregateCompletionPath(workDir string, groupID int64) string { + return filepath.Join(workDir, storachaAggregateCompleteDir, fmt.Sprintf("%0*d.json", storachaAggregateGroupIDWidth, groupID)) +} + +func storachaAggregateTempFilePath(workDir string, groupID int64) string { + return filepath.Join(workDir, storachaAggregateTempDir, fmt.Sprintf("%0*d.car.tmp", storachaAggregateGroupIDWidth, groupID)) +} + +func storachaAggregateBucketFileName(size abi.PaddedPieceSize) string { + return fmt.Sprintf("%d.jsonl", uint64(size)) +} + +func storachaAggregateBucketSizesAscending() []abi.PaddedPieceSize { + // Filecoin piece sizes are powers of two. For a 1-GiB aggregate target this + // creates bucket names from 128.jsonl through 1073741824.jsonl. + sizes := make([]abi.PaddedPieceSize, 0, 24) + for size := abi.PaddedPieceSize(128); size <= storachaAggregatePieceSize; size <<= 1 { + sizes = append(sizes, size) + } + return sizes +} + +func storachaAggregateBucketSizesDescending() []abi.PaddedPieceSize { + // Grouping consumes largest buckets first, but raw finalization/checksums are + // easier to reason about in ascending filename order. + ascending := storachaAggregateBucketSizesAscending() + descending := make([]abi.PaddedPieceSize, 0, len(ascending)) + for i := len(ascending) - 1; i >= 0; i-- { + descending = append(descending, ascending[i]) + } + return descending +} + +func storachaAggregateIsBucketSize(size abi.PaddedPieceSize) bool { + if size < 128 || size > storachaAggregatePieceSize { + return false + } + if err := size.Validate(); err != nil { + return false + } + return true +} + +func writeJSONLine(writer *bufio.Writer, value any) (int, error) { + line, err := json.Marshal(value) + if err != nil { + return 0, err + } + line = append(line, '\n') + return writer.Write(line) +} + +func writeAggregatePiecesResultFromWork(path, workDir string, runErr error) error { + // The result is reconstructed from completion markers, not from in-memory + // state. That makes it safe to report completed aggregates after retries, + // crashes, or a final failure in a later group. + if workDir == "" { + out := aggregatePiecesOutput{Aggregates: []aggregates{}} + if runErr != nil { + out.Error = runErr.Error() + } + return writeAggregatePiecesResult(path, out) + } + + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return xerrors.Errorf("creating aggregate result directory: %w", err) + } + + tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating aggregate result temp file: %w", err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + writer := bufio.NewWriterSize(file, 1024*1024) + if _, err := writer.WriteString("{\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if runErr != nil { + errJSON, err := json.Marshal(runErr.Error()) + if err != nil { + _ = file.Close() + return xerrors.Errorf("marshalling aggregate result error: %w", err) + } + if _, err := writer.WriteString(" \"error\": "); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if _, err := writer.Write(errJSON); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if _, err := writer.WriteString(",\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if _, err := writer.WriteString(" \"aggregates\": ["); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + + first := true + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + groupsFile, err := os.Open(groupsPath) + if err == nil { + // Stream groups.jsonl instead of loading it. There may be many aggregate + // groups, but only completed groups produce result entries. + reader := bufio.NewReaderSize(groupsFile, 1024*1024) + for { + line, ok, readErr := readJSONLLine(reader) + if readErr != nil { + if runErr != nil { + break + } + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("reading aggregate groups for result: %w", readErr) + } + if !ok { + break + } + + var group storachaAggregateGroup + if err := json.Unmarshal(line, &group); err != nil { + if runErr != nil { + break + } + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("decoding aggregate group for result: %w", err) + } + + complete, ok, err := readCompletedStorachaAggregateGroup(workDir, group) + if err != nil { + if runErr != nil { + continue + } + _ = groupsFile.Close() + _ = file.Close() + return err + } + if !ok { + continue + } + + result := struct { + PieceCID string `json:"piece_cid"` + SubPieces []string `json:"sub_pieces"` + Count int `json:"count"` + }{ + PieceCID: complete.PieceCID, + SubPieces: complete.SubPieces, + Count: complete.Count, + } + resultJSON, err := json.Marshal(result) + if err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("marshalling aggregate result: %w", err) + } + + if first { + if _, err := writer.WriteString("\n "); err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + first = false + } else { + if _, err := writer.WriteString(",\n "); err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if _, err := writer.Write(resultJSON); err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if err := groupsFile.Close(); err != nil { + _ = file.Close() + return xerrors.Errorf("closing aggregate groups for result: %w", err) + } + } else if !os.IsNotExist(err) { + _ = file.Close() + return xerrors.Errorf("opening aggregate groups for result: %w", err) + } + + if first { + if _, err := writer.WriteString("]\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } else { + if _, err := writer.WriteString("\n ]\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if _, err := writer.WriteString("}\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if err := writer.Flush(); err != nil { + _ = file.Close() + return xerrors.Errorf("flushing aggregate result: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return xerrors.Errorf("syncing aggregate result: %w", err) + } + if err := file.Close(); err != nil { + return xerrors.Errorf("closing aggregate result: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return xerrors.Errorf("renaming aggregate result: %w", err) + } + removeTmp = false + return syncDir(filepath.Dir(path)) +} + +func writeAggregatePiecesResult(path string, out aggregatePiecesOutput) error { + if out.Aggregates == nil { + out.Aggregates = []aggregates{} + } + if err := writeJSONFileAtomic(path, out); err != nil { + return xerrors.Errorf("writing aggregate result: %w", err) + } + return nil +} + +func writeJSONFileAtomic(path string, value any) error { + // All small state files use the same tmp-file protocol: write, fsync, close, + // rename, then fsync the directory so the rename itself survives a crash. + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return xerrors.Errorf("creating directory for %s: %w", path, err) + } + + tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating temp JSON file %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(value) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(encodeErr, syncErr, closeErr); err != nil { + return xerrors.Errorf("writing temp JSON file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, path); err != nil { + return xerrors.Errorf("renaming temp JSON file %s to %s: %w", tmpPath, path, err) + } + removeTmp = false + return syncDir(filepath.Dir(path)) +} + +func syncDir(path string) error { + // Directory fsync is what makes a preceding rename durable on filesystems + // that otherwise may persist file contents without persisting the directory + // entry update. + dir, err := os.Open(path) + if err != nil { + return xerrors.Errorf("opening directory %s for sync: %w", path, err) + } + defer func() { + _ = dir.Close() + }() + if err := dir.Sync(); err != nil { + return xerrors.Errorf("syncing directory %s: %w", path, err) + } + return nil +} diff --git a/cmd/curio/storacha-migration-aggregate_test.go b/cmd/curio/storacha-migration-aggregate_test.go new file mode 100644 index 000000000..a7e296180 --- /dev/null +++ b/cmd/curio/storacha-migration-aggregate_test.go @@ -0,0 +1,493 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/market/mk20" +) + +type storachaAggregateTestPiece struct { + storachaPieceFixture + parkedID int64 +} + +func TestStorachaAggregateDedupeAndCompletedMarkerResume(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2, pieces[0].cidV2) + + workDir, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + manifest := readStorachaAggregateManifestForTest(t, workDir) + require.Equal(t, int64(3), manifest.RawBuckets.Records) + require.Equal(t, int64(2), manifest.DedupedBuckets.Records) + require.Equal(t, int64(2), manifest.Groups.Pieces) + + group := readStorachaAggregateGroupForTest(t, workDir, 0) + require.Len(t, group.Pieces, 2) + complete := assertStorachaAggregateImported(t, env, workDir, group) + + // The completion marker is the resume boundary after a group is fully + // imported. Remove the source subpieces to prove a rerun trusts the marker + // instead of trying to read source bytes again. + for _, piece := range pieces { + require.NoError(t, os.Remove(storachaFinalPiecePath(env.sourceDir, piece.parkedID))) + } + + _, err = runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + require.Equal(t, complete, readStorachaAggregateCompletionForTest(t, workDir, group)) +} + +func TestStorachaAggregateRawBucketCursorTruncatesTornTail(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + inputSHA256, _, err := hashFileSHA256(inputPath) + require.NoError(t, err) + + workDir := filepath.Join(env.targetDir, storachaAggregateWorkDir, inputSHA256) + rawDir := filepath.Join(workDir, storachaAggregateRawBucketDir) + require.NoError(t, os.MkdirAll(rawDir, 0755)) + + firstRecord := storachaAggregateBucketRecordForTest(pieces[0]) + firstRecordLine, err := json.Marshal(firstRecord) + require.NoError(t, err) + firstRecordLine = append(firstRecordLine, '\n') + + rawBucketName := storachaAggregateBucketFileName(abi.PaddedPieceSize(pieces[0].info.Size)) + rawBucketPath := filepath.Join(rawDir, rawBucketName) + writeFile(t, rawBucketPath, append(firstRecordLine, []byte(`{"piece_cid_v2":"torn"`)...)) + require.NoError(t, writeJSONFileAtomic(filepath.Join(workDir, storachaAggregateBucketCursor), storachaAggregateBucketCursorState{ + InputSHA256: inputSHA256, + InputOffset: int64(len(pieces[0].cidV2) + 1), + InputLine: 1, + Records: 1, + BucketOffsets: map[string]int64{ + rawBucketName: int64(len(firstRecordLine)), + }, + })) + + actualWorkDir, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + require.Equal(t, workDir, actualWorkDir) + + rawBytes, err := os.ReadFile(rawBucketPath) + require.NoError(t, err) + require.NotContains(t, string(rawBytes), "torn") + + manifest := readStorachaAggregateManifestForTest(t, workDir) + require.Equal(t, int64(2), manifest.RawBuckets.Records) + require.Equal(t, int64(2), manifest.DedupedBuckets.Records) + assertStorachaAggregateImported(t, env, workDir, readStorachaAggregateGroupForTest(t, workDir, 0)) +} + +func TestStorachaAggregateIgnoresExistingTempFile(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath := storachaAggregateTempFilePath(workDir, group.GroupID) + writeFile(t, tempPath, []byte("leftover temp data")) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + assertMissing(t, tempPath) + assertStorachaAggregateImported(t, env, workDir, group) +} + +func TestStorachaAggregateRecoversFromStagingFile(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + stagingDir := storachaAggregateStagingDir(env.targetDir) + require.NoError(t, os.MkdirAll(stagingDir, 0755)) + stagingPath := storachaAggregateStagingPath(env.targetDir, verified.PieceCIDV2) + require.NoError(t, os.Rename(tempPath, stagingPath)) + require.NoError(t, syncDir(stagingDir)) + require.NoError(t, writeStorachaAggregateStagedMarker(env.targetDir, verified.PieceCIDV2)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + assertMissing(t, stagingPath) + assertMissing(t, storachaAggregateStagedMarkerPath(env.targetDir, verified.PieceCIDV2)) + complete := assertStorachaAggregateImported(t, env, workDir, group) + require.Equal(t, verified.PieceCIDV2.String(), complete.PieceCID) +} + +func TestStorachaAggregateRecoversFromFinalFileAndFixesDB(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + pieceID := insertAggregateParkedPiece(t, env, verified, false) + assertStorachaRefs(t, env, pieceID, 0) + assertSectorLocation(t, env, pieceID, 0) + + finalPath := storachaFinalPiecePath(env.targetDir, pieceID) + require.NoError(t, os.Rename(tempPath, finalPath)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, group.PieceCIDV1, 1) + assertSectorLocation(t, env, pieceID, 1) + + complete := assertStorachaAggregateImported(t, env, workDir, group) + require.Equal(t, verified.PieceCIDV2.String(), complete.PieceCID) +} + +func TestStorachaAggregateUncommittedStagingRequiresManualCleanup(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + require.NoError(t, os.Remove(tempPath)) + + stagingDir := storachaAggregateStagingDir(env.targetDir) + require.NoError(t, os.MkdirAll(stagingDir, 0755)) + stagingPath := storachaAggregateStagingPath(env.targetDir, verified.PieceCIDV2) + writeFile(t, stagingPath, bytes.Repeat([]byte{0x7f}, int(verified.RawSize))) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.ErrorContains(t, err, "uncommitted aggregate staging file already exists") + require.ErrorContains(t, err, "cleanup manually") + assertMissing(t, storachaAggregateCompletionPath(workDir, group.GroupID)) +} + +func TestStorachaAggregateStagedMarkerMissingFileRequiresManualCleanup(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + require.NoError(t, os.Remove(tempPath)) + require.NoError(t, writeStorachaAggregateStagedMarker(env.targetDir, verified.PieceCIDV2)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.ErrorContains(t, err, "staged marker") + require.ErrorContains(t, err, "is missing") + require.ErrorContains(t, err, "cleanup manually") + assertMissing(t, storachaAggregateCompletionPath(workDir, group.GroupID)) +} + +func TestStorachaAggregateFinalSizeMismatchRequiresManualCleanup(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + require.NoError(t, os.Remove(tempPath)) + + pieceID := insertAggregateParkedPiece(t, env, verified, false) + writeFile(t, storachaFinalPiecePath(env.targetDir, pieceID), bytes.Repeat([]byte{0x42}, int(verified.RawSize)-1)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.ErrorContains(t, err, "size mismatch") + require.ErrorContains(t, err, "cleanup manually") + assertMissing(t, storachaAggregateCompletionPath(workDir, group.GroupID)) + require.False(t, parkedPieceByID(t, env, pieceID).complete) +} + +func withSmallStorachaAggregateSize(t *testing.T) { + t.Helper() + + previous := storachaAggregatePieceSize + storachaAggregatePieceSize = abi.PaddedPieceSize(4 << 20) + t.Cleanup(func() { + storachaAggregatePieceSize = previous + }) +} + +func prepareStorachaAggregateSourcePieces(t *testing.T, env storachaMigrationTestEnv, count int) []storachaAggregateTestPiece { + t.Helper() + + pieces := make([]storachaAggregateTestPiece, 0, count) + for i := 0; i < count; i++ { + fx := newValidStorachaAggregatePieceFixture(t, byte(i+1)) + pieceID := insertParkedPiece(t, env, fx, true, true, nil) + writeFile(t, storachaFinalPiecePath(env.sourceDir, pieceID), fx.data) + pieces = append(pieces, storachaAggregateTestPiece{ + storachaPieceFixture: fx, + parkedID: pieceID, + }) + } + return pieces +} + +func newValidStorachaAggregatePieceFixture(t *testing.T, seed byte) storachaPieceFixture { + t.Helper() + + data := bytes.Repeat([]byte{seed}, 127) + cp := new(commp.Calc) + defer cp.Reset() + n, err := io.Copy(cp, bytes.NewReader(data)) + require.NoError(t, err) + + digest, paddedSize, err := cp.Digest() + require.NoError(t, err) + + pcidV2, err := commcid.DataCommitmentToPieceCidv2(digest, uint64(n)) + require.NoError(t, err) + + info, err := mk20.GetPieceInfo(pcidV2) + require.NoError(t, err) + require.Equal(t, uint64(n), info.RawSize) + require.Equal(t, abi.PaddedPieceSize(paddedSize), info.Size) + + return storachaPieceFixture{ + cidV2: pcidV2.String(), + fileName: pcidV2.String() + ".car", + info: info, + data: data, + } +} + +func writeStorachaAggregateInput(t *testing.T, pieceCIDs ...string) string { + t.Helper() + + inputPath := filepath.Join(t.TempDir(), "aggregate-input.txt") + writeFile(t, inputPath, []byte(strings.Join(pieceCIDs, "\n")+"\n")) + return inputPath +} + +func storachaAggregateSortPath(t *testing.T) string { + t.Helper() + + sortPath, err := exec.LookPath("sort") + require.NoError(t, err) + return sortPath +} + +func prepareStorachaAggregateGroupsForTest(t *testing.T, env storachaMigrationTestEnv, inputPath string) (string, storachaAggregateGroup) { + t.Helper() + + inputSHA256, inputSize, err := hashFileSHA256(inputPath) + require.NoError(t, err) + + workDir := filepath.Join(env.targetDir, storachaAggregateWorkDir, inputSHA256) + require.NoError(t, os.MkdirAll(workDir, 0755)) + + manifest, err := loadStorachaAggregateManifest(workDir, inputSHA256, inputSize) + require.NoError(t, err) + require.NoError(t, buildRawStorachaAggregateBuckets(env.ctx, env.db, inputPath, inputSHA256, workDir, manifest)) + require.NoError(t, dedupeStorachaAggregateBuckets(env.ctx, storachaAggregateSortPath(t), workDir, manifest)) + require.NoError(t, buildStorachaAggregateGroups(workDir, manifest)) + + return workDir, readStorachaAggregateGroupForTest(t, workDir, 0) +} + +func createStorachaAggregateTempFileForTest(t *testing.T, env storachaMigrationTestEnv, workDir string, group storachaAggregateGroup) (string, storachaVerifiedAggregatePiece) { + t.Helper() + + expectedCIDV1, err := cid.Parse(group.PieceCIDV1) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(workDir, storachaAggregateTempDir), 0755)) + + tempPath, verified, err := createStorachaAggregateTempFile(env.sourceDir, workDir, group, expectedCIDV1, abi.PaddedPieceSize(group.PaddedSize)) + require.NoError(t, err) + require.Equal(t, group.PieceCIDV1, verified.PieceCIDV1.String()) + require.Equal(t, abi.PaddedPieceSize(group.PaddedSize), verified.PaddedSize) + return tempPath, verified +} + +func readStorachaAggregateManifestForTest(t *testing.T, workDir string) storachaAggregateWorkManifest { + t.Helper() + + mb, err := os.ReadFile(filepath.Join(workDir, storachaAggregateManifestFile)) + require.NoError(t, err) + + var manifest storachaAggregateWorkManifest + require.NoError(t, json.Unmarshal(mb, &manifest)) + return manifest +} + +func readStorachaAggregateGroupForTest(t *testing.T, workDir string, groupID int64) storachaAggregateGroup { + t.Helper() + + file, err := os.Open(filepath.Join(workDir, storachaAggregateGroupsFile)) + require.NoError(t, err) + defer func() { + _ = file.Close() + }() + + reader := bufio.NewReader(file) + for { + line, ok, err := readJSONLLine(reader) + require.NoError(t, err) + require.True(t, ok, "group %d not found", groupID) + + var group storachaAggregateGroup + require.NoError(t, json.Unmarshal(line, &group)) + if group.GroupID == groupID { + return group + } + } +} + +func readStorachaAggregateCompletionForTest(t *testing.T, workDir string, group storachaAggregateGroup) storachaAggregateCompletion { + t.Helper() + + complete, ok, err := readCompletedStorachaAggregateGroup(workDir, group) + require.NoError(t, err) + require.True(t, ok) + return complete +} + +func assertStorachaAggregateImported(t *testing.T, env storachaMigrationTestEnv, workDir string, group storachaAggregateGroup) storachaAggregateCompletion { + t.Helper() + + complete := readStorachaAggregateCompletionForTest(t, workDir, group) + pcidV2, err := cid.Parse(complete.PieceCID) + require.NoError(t, err) + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + require.NoError(t, err) + require.Equal(t, group.PieceCIDV1, pcidV1.String()) + require.Equal(t, complete.RawSize, rawSize) + + var rows []storachaFinalRecoveryRow + err = env.db.Select(env.ctx, &rows, ` + SELECT id, piece_cid, piece_padded_size, piece_raw_size + FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND piece_raw_size = $3 + AND long_term = TRUE + AND cleanup_task_id IS NULL + ORDER BY id`, group.PieceCIDV1, int64(group.PaddedSize), int64(complete.RawSize)) + require.NoError(t, err) + require.Len(t, rows, 1) + + pp := parkedPieceByID(t, env, rows[0].ID) + require.True(t, pp.complete) + assertStorachaRefs(t, env, rows[0].ID, 1) + assertPDPRefs(t, env, rows[0].ID, group.PieceCIDV1, 1) + assertSectorLocation(t, env, rows[0].ID, 1) + + _, err = verifyStorachaAggregateFileForTest(storachaFinalPiecePath(env.targetDir, rows[0].ID), pcidV1, abi.PaddedPieceSize(group.PaddedSize)) + require.NoError(t, err) + return complete +} + +func verifyStorachaAggregateFileForTest(path string, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (storachaVerifiedAggregatePiece, error) { + file, err := os.Open(path) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + defer func() { + _ = file.Close() + }() + + info, err := file.Stat() + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + if info.IsDir() { + return storachaVerifiedAggregatePiece{}, os.ErrInvalid + } + + cp := new(commp.Calc) + defer cp.Reset() + n, err := io.Copy(cp, file) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + + digest, actualPaddedSize, err := cp.Digest() + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + if abi.PaddedPieceSize(actualPaddedSize) != expectedPaddedSize { + return storachaVerifiedAggregatePiece{}, os.ErrInvalid + } + + actualCIDV1, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + if !actualCIDV1.Equals(expectedCIDV1) { + return storachaVerifiedAggregatePiece{}, os.ErrInvalid + } + + actualCIDV2, err := commcid.DataCommitmentToPieceCidv2(digest, uint64(n)) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + + return storachaVerifiedAggregatePiece{ + PieceCIDV2: actualCIDV2, + PieceCIDV1: actualCIDV1, + RawSize: uint64(n), + PaddedSize: abi.PaddedPieceSize(actualPaddedSize), + }, nil +} + +func insertAggregateParkedPiece(t *testing.T, env storachaMigrationTestEnv, verified storachaVerifiedAggregatePiece, complete bool) int64 { + t.Helper() + + var pieceID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, complete, long_term, skip) + VALUES ($1, $2, $3, $4, TRUE, TRUE) + RETURNING id + `, verified.PieceCIDV1.String(), int64(verified.PaddedSize), int64(verified.RawSize), complete).Scan(&pieceID) + require.NoError(t, err) + return pieceID +} + +func storachaAggregateBucketRecordForTest(piece storachaAggregateTestPiece) storachaBucketRecord { + return storachaBucketRecord{ + PieceCIDV2: piece.cidV2, + PieceID: piece.parkedID, + PieceCIDV1: piece.info.PieceCIDV1.String(), + RawSize: piece.info.RawSize, + PaddedSize: uint64(piece.info.Size), + } +} diff --git a/cmd/curio/toolbox.go b/cmd/curio/toolbox.go index 5aa7ee096..9488b4327 100644 --- a/cmd/curio/toolbox.go +++ b/cmd/curio/toolbox.go @@ -41,6 +41,7 @@ var toolboxCmd = &cli.Command{ downgradeCmd, fixBoostMigrationCmd, importPiecesCmd, + aggregatePiecesCmd, }, } From 01d8259a40e997acca8c42b4c22773e0667b64c2 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 22 Jul 2026 17:43:31 +0400 Subject: [PATCH 8/9] separate aggregate piece_ref, no pdp ref for aggregate --- cmd/curio/storacha-migration-aggregate.go | 7 +-- .../storacha-migration-aggregate_test.go | 11 +++-- cmd/curio/storacha-migration.go | 47 ++++++++++++------- cmd/curio/storacha-migration_test.go | 12 ++++- 4 files changed, 52 insertions(+), 25 deletions(-) diff --git a/cmd/curio/storacha-migration-aggregate.go b/cmd/curio/storacha-migration-aggregate.go index 712ca4aeb..56c5433c6 100644 --- a/cmd/curio/storacha-migration-aggregate.go +++ b/cmd/curio/storacha-migration-aggregate.go @@ -34,6 +34,7 @@ import ( ) const storachaAggregateDefaultPieceSize = abi.PaddedPieceSize(1 << 30) +const storachaMigrationAggregateDataURL = "storacha-migration-aggregate" // storachaAggregatePieceSize is a variable only so tests can exercise the // recovery paths without writing 1 GiB aggregate files. Production uses the @@ -290,7 +291,7 @@ var aggregatePiecesCmd = &cli.Command{ return err } - si := paths.NewDBIndex(curioalerting.NewAlertingSystem(), db) + si := paths.NewDBIndex(curioalerting.NewAlertingSystem(db), db) workDir, err = runAggregatePieces(ctx, db, si, storageID, sortPath, inputPath, sourcePath, targetPath) if err != nil { @@ -1771,7 +1772,7 @@ func recoverStorachaAggregateFromFinal(ctx context.Context, db *harmonydb.DB, si return false, err } - imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row) + imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row, false) if err != nil { return false, err } @@ -2045,7 +2046,7 @@ func importStagedStorachaAggregateFile(ctx context.Context, db *harmonydb.DB, si return xerrors.Errorf("aggregate staging file %s size mismatch: expected %d, got %d; cleanup manually before retrying", stagingPath, verified.RawSize, info.Size()) } - result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath) + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath, false) if err != nil { return err } diff --git a/cmd/curio/storacha-migration-aggregate_test.go b/cmd/curio/storacha-migration-aggregate_test.go index a7e296180..4d5ef03d1 100644 --- a/cmd/curio/storacha-migration-aggregate_test.go +++ b/cmd/curio/storacha-migration-aggregate_test.go @@ -156,6 +156,7 @@ func TestStorachaAggregateRecoversFromFinalFileAndFixesDB(t *testing.T) { tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) pieceID := insertAggregateParkedPiece(t, env, verified, false) assertStorachaRefs(t, env, pieceID, 0) + assertStorachaRefsWithDataURL(t, env, pieceID, storachaMigrationAggregateDataURL, 0) assertSectorLocation(t, env, pieceID, 0) finalPath := storachaFinalPiecePath(env.targetDir, pieceID) @@ -166,8 +167,9 @@ func TestStorachaAggregateRecoversFromFinalFileAndFixesDB(t *testing.T) { pp := parkedPieceByID(t, env, pieceID) require.True(t, pp.complete) - assertStorachaRefs(t, env, pieceID, 1) - assertPDPRefs(t, env, pieceID, group.PieceCIDV1, 1) + assertStorachaRefs(t, env, pieceID, 0) + assertStorachaRefsWithDataURL(t, env, pieceID, storachaMigrationAggregateDataURL, 1) + assertPDPRefs(t, env, pieceID, group.PieceCIDV1, 0) assertSectorLocation(t, env, pieceID, 1) complete := assertStorachaAggregateImported(t, env, workDir, group) @@ -407,8 +409,9 @@ func assertStorachaAggregateImported(t *testing.T, env storachaMigrationTestEnv, pp := parkedPieceByID(t, env, rows[0].ID) require.True(t, pp.complete) - assertStorachaRefs(t, env, rows[0].ID, 1) - assertPDPRefs(t, env, rows[0].ID, group.PieceCIDV1, 1) + assertStorachaRefs(t, env, rows[0].ID, 0) + assertStorachaRefsWithDataURL(t, env, rows[0].ID, storachaMigrationAggregateDataURL, 1) + assertPDPRefs(t, env, rows[0].ID, group.PieceCIDV1, 0) assertSectorLocation(t, env, rows[0].ID, 1) _, err = verifyStorachaAggregateFileForTest(storachaFinalPiecePath(env.targetDir, rows[0].ID), pcidV1, abi.PaddedPieceSize(group.PaddedSize)) diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go index a03c70cc5..cb01498ac 100644 --- a/cmd/curio/storacha-migration.go +++ b/cmd/curio/storacha-migration.go @@ -166,7 +166,7 @@ var importPiecesCmd = &cli.Command{ return err } - si := paths.NewDBIndex(curioalerting.NewAlertingSystem(), db) + si := paths.NewDBIndex(curioalerting.NewAlertingSystem(db), db) out, err = runImportPieces(ctx, db, si, storageID, sourcePath, targetPath, cctx.Int("batch-size")) if err != nil { @@ -217,7 +217,7 @@ func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex continue } - result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, filepath.Join(staging, entry.Name())) + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, filepath.Join(staging, entry.Name()), true) if err != nil { return out, err } @@ -276,7 +276,7 @@ func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex return out, xerrors.Errorf("renaming old path: %w", err) } - result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath) + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath, true) if err != nil { return out, err } @@ -334,13 +334,15 @@ func storachaPieceInfoFromFileName(name string) (*mk20.PieceInfo, bool, error) { // importStagedStorachaPiece finishes one target/storacha-staging/.car. // It first creates or reuses parked_pieces/refs in a committed DB transaction. +// makePDPPieceRef keeps import-pieces behavior unchanged while letting aggregate +// imports use the aggregate migration data_url and skip the public PDP piece ref. // If an existing incomplete parked_piece still has a live task, it returns // Imported=false and leaves the staged file untouched for a later retry. // Otherwise it consumes the staged file: remove it if the final // target/piece/s-t00- already exists, or rename it to that final path. It // then declares the piece location and marks parked_pieces.complete unless the // row was already complete. -func importStagedStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, stagingPath string) (storachaImportResult, error) { +func importStagedStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, stagingPath string, makePDPPieceRef bool) (storachaImportResult, error) { info, err := os.Stat(stagingPath) if err != nil { return storachaImportResult{}, xerrors.Errorf("checking staging file: %w", err) @@ -383,7 +385,7 @@ func importStagedStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.S return true, nil } - if err := ensureStorachaRefsTx(tx, parkedPieceID, pi); err != nil { + if err := ensureStorachaRefsTx(tx, parkedPieceID, pi, makePDPPieceRef); err != nil { return false, err } @@ -474,7 +476,7 @@ func recoverFinalStorachaPieces(ctx context.Context, db *harmonydb.DB, si paths. break } - imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row) + imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row, true) if err != nil { return err } @@ -510,7 +512,7 @@ func storachaPieceCIDV2FromRow(row storachaFinalRecoveryRow) (string, error) { // DB row, reuses the same claim/ref checks as staged import, then declares the // final file and marks the parked piece complete. It returns false when the // final file is missing or a live task still owns the parked_piece. -func recoverFinalStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath string, row storachaFinalRecoveryRow) (bool, error) { +func recoverFinalStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath string, row storachaFinalRecoveryRow, makePDPPieceRef bool) (bool, error) { finalPath := storachaFinalPiecePath(targetPath, row.ID) info, err := os.Stat(finalPath) if os.IsNotExist(err) { @@ -542,7 +544,7 @@ func recoverFinalStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.S if !state.UseStagedFile { return true, nil } - if err := ensureStorachaRefsTx(tx, row.ID, pi); err != nil { + if err := ensureStorachaRefsTx(tx, row.ID, pi, makePDPPieceRef); err != nil { return false, err } return true, nil @@ -617,12 +619,15 @@ func claimExistingStorachaPieceTx(tx *harmonydb.Tx, parkedPieceID int64) (storac return state, nil } -// ensureStorachaRefsTx enforces the intended one-to-one shape for this import. -// It locks storacha-migration parked_piece_refs for this parked piece, fails if -// more than one exists, inserts the missing ref if needed, then does the same -// for pdp_piecerefs through the unique piece_ref relation. Existing PDP refs -// must already point at the same service and piece CID. -func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceInfo) error { +// ensureStorachaRefsTx enforces the intended one-to-one shape for this migration +// mode. Direct imports create storacha-migration refs and PDP refs; aggregate +// imports create storacha-migration-aggregate refs and intentionally skip PDP refs. +func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceInfo, makePDPPieceRef bool) error { + dataURL := storachaMigrationAggregateDataURL + if makePDPPieceRef { + dataURL = storachaMigrationDataURL + } + var refs []struct { RefID int64 `db:"ref_id"` } @@ -633,7 +638,7 @@ func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceI AND data_url = $2 AND long_term = TRUE ORDER BY ref_id - FOR UPDATE`, parkedPieceID, storachaMigrationDataURL) + FOR UPDATE`, parkedPieceID, dataURL) if err != nil { return xerrors.Errorf("query storacha parked piece refs: %w", err) } @@ -648,18 +653,26 @@ func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceI err = tx.QueryRow(` INSERT INTO parked_piece_refs (piece_id, data_url, long_term) VALUES ($1, $2, TRUE) - RETURNING ref_id`, parkedPieceID, storachaMigrationDataURL).Scan(&pieceRef) + RETURNING ref_id`, parkedPieceID, dataURL).Scan(&pieceRef) if err != nil { return xerrors.Errorf("insert storacha parked piece ref: %w", err) } } + if !makePDPPieceRef { + return nil + } + + return ensurePDPPieceRef(tx, pieceRef, pi) +} + +func ensurePDPPieceRef(tx *harmonydb.Tx, pieceRef int64, pi *mk20.PieceInfo) error { var pdpRefs []struct { ID int64 `db:"id"` Service string `db:"service"` PieceCID string `db:"piece_cid"` } - err = tx.Select(&pdpRefs, ` + err := tx.Select(&pdpRefs, ` SELECT id, service, piece_cid FROM pdp_piecerefs WHERE piece_ref = $1 diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go index 1fe9a1ca8..bdf5a1043 100644 --- a/cmd/curio/storacha-migration_test.go +++ b/cmd/curio/storacha-migration_test.go @@ -666,6 +666,11 @@ func parkedPieceByID(t *testing.T, env storachaMigrationTestEnv, pieceID int64) func storachaRefIDs(t *testing.T, env storachaMigrationTestEnv, pieceID int64) []int64 { t.Helper() + return storachaRefIDsForDataURL(t, env, pieceID, storachaMigrationDataURL) +} + +func storachaRefIDsForDataURL(t *testing.T, env storachaMigrationTestEnv, pieceID int64, dataURL string) []int64 { + t.Helper() var rows []struct { RefID int64 `db:"ref_id"` @@ -677,7 +682,7 @@ func storachaRefIDs(t *testing.T, env storachaMigrationTestEnv, pieceID int64) [ AND data_url = $2 AND long_term = TRUE ORDER BY ref_id - `, pieceID, storachaMigrationDataURL) + `, pieceID, dataURL) require.NoError(t, err) out := make([]int64, 0, len(rows)) @@ -707,6 +712,11 @@ func assertStorachaRefs(t *testing.T, env storachaMigrationTestEnv, pieceID int6 require.Len(t, storachaRefIDs(t, env, pieceID), expected) } +func assertStorachaRefsWithDataURL(t *testing.T, env storachaMigrationTestEnv, pieceID int64, dataURL string, expected int) { + t.Helper() + require.Len(t, storachaRefIDsForDataURL(t, env, pieceID, dataURL), expected) +} + func assertPDPRefs(t *testing.T, env storachaMigrationTestEnv, pieceID int64, pieceCID string, expected int) { t.Helper() refs := pdpRefsForPiece(t, env, pieceID) From e37bbea5b751b71f09741c5887c003ec4d85e409 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 22 Jul 2026 18:18:24 +0400 Subject: [PATCH 9/9] make gen --- documentation/en/curio-cli/curio.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index 28ef9fb94..a077f0765 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -1385,6 +1385,7 @@ COMMANDS: downgrade Downgrade a cluster's database to a previous software version. fix-boost-migration Fix Boost migration import-pieces Imports already existing pieces from storage to piece park system + aggregate-pieces Aggregates already existing pieces from storage into PODSI and park them help, h Shows a list of commands or help for one command OPTIONS: @@ -1453,6 +1454,22 @@ OPTIONS: --help, -h show help ``` +### curio toolbox aggregate-pieces +``` +NAME: + curio toolbox aggregate-pieces - Aggregates already existing pieces from storage into PODSI and park them + +USAGE: + curio toolbox aggregate-pieces [command options] + +OPTIONS: + --source value path to read the pieces from + --target value path to storage directory in Curio's attached permanent storage + --result value path to write the JSON result + --input value path to read the CIDs from + --help, -h show help +``` + ## curio batch ``` NAME: