diff --git a/cmd/curio/storacha-migration-aggregate.go b/cmd/curio/storacha-migration-aggregate.go new file mode 100644 index 000000000..56c5433c6 --- /dev/null +++ b/cmd/curio/storacha-migration-aggregate.go @@ -0,0 +1,2511 @@ +package main + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/ipfs/go-cid" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-data-segment/datasegment" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/alertmanager/curioalerting" + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" +) + +const storachaAggregateDefaultPieceSize = abi.PaddedPieceSize(1 << 30) +const storachaMigrationAggregateDataURL = "storacha-migration-aggregate" + +// storachaAggregatePieceSize is a variable only so tests can exercise the +// recovery paths without writing 1 GiB aggregate files. Production uses the +// default 1 GiB padded aggregate size. +var storachaAggregatePieceSize = storachaAggregateDefaultPieceSize + +// aggregate-pieces is intentionally a file-backed pipeline, not a task table. +// +// The input is treated as read-only and is identified by its SHA256. That hash +// gives every distinct input file a distinct work directory, so changing the +// input never silently resumes with stale bucket/group data. +// +// The durable stages are: +// 1. raw buckets: stream the input, hydrate parked_piece ids from the DB, and +// append full records to padded-size bucket files with a cursor for resume. +// 2. deduped buckets: use the OS sort command per bucket, then drop adjacent +// duplicate PieceCIDv2 records. +// 3. groups.jsonl: write one atomic grouping file. This is the replay plan; a +// partial regroup would change aggregate membership and is not resumable. +// 4. complete/*.json: process each group into a real aggregate file and write +// a per-group commit marker after the piece is imported. +// +// Completed JSONL stage files are validated by size, record count, and SHA256 +// before they are trusted on retry. Aggregate body files use a different +// boundary: temp files are uncommitted scratch and can be overwritten, staged +// files are trusted only after staged/ exists, and final files are +// trusted from their DB row plus final path/size. After the staged marker exists +// we do not run CommP again; a rename or failed fsync can lose a directory entry +// but cannot change already-written bytes into another valid file. +type aggregatePiecesOutput struct { + Error string `json:"error,omitempty"` + Aggregates []aggregates `json:"aggregates"` +} + +// aggregates is the public result contract for this command. It reports only +// aggregates that reached a completion marker, so callers can safely consume a +// partial result even when the command exits with an error. +type aggregates struct { + PieceCid cid.Cid `json:"piece_cid"` + SubPieces []cid.Cid `json:"sub_pieces"` + Count int `json:"count"` +} + +// pieceDetails is the transient hydrated form of one input line. The JSONL +// bucket records below are the durable form; this type exists only while a DB +// hydration batch is in memory. +type pieceDetails struct { + PieceCidV2 cid.Cid + PieceCID cid.Cid `db:"piece_cid"` + PaddedSize abi.PaddedPieceSize `db:"piece_padded_size"` + RawSize uint64 + ID int64 `db:"id"` +} + +const ( + storachaAggregateManifestVersion = 1 + storachaAggregateHydrateBatch = 5000 + storachaAggregateGroupIDWidth = 20 + + storachaAggregateWorkDir = "storacha-aggregate-work" + storachaAggregateManifestFile = "manifest.json" + storachaAggregateBucketCursor = "bucket-cursor.json" + storachaAggregateRawBucketDir = "buckets.raw" + storachaAggregateDedupeDir = "buckets" + storachaAggregateGroupsFile = "groups.jsonl" + storachaAggregateCompleteDir = "complete" + storachaAggregateTempDir = "aggregate-tmp" + storachaAggregateStagingDirName = "storacha-aggregate-staging" + storachaAggregateStagedDirName = "staged" +) + +// storachaBucketRecord is the durable "full hydrated record" for one input +// piece. PieceCIDV2 is deliberately the first JSON field: OS sort compares +// whole JSONL lines, so putting the dedupe key first makes equal PieceCIDv2 +// records adjacent without a custom sorter or an in-memory seen set. +type storachaBucketRecord struct { + PieceCIDV2 string `json:"piece_cid_v2"` + PieceID int64 `json:"piece_id"` + PieceCIDV1 string `json:"piece_cid_v1"` + RawSize uint64 `json:"raw_size"` + PaddedSize uint64 `json:"padded_size"` +} + +// storachaAggregateGroup is the single atomic grouping artifact. Once this +// file exists, aggregate creation is just replaying deterministic work. That is +// important because a partial regrouping would change which subpieces belong to +// which aggregate and make resume semantics ambiguous. +type storachaAggregateGroup struct { + GroupID int64 `json:"group_id"` + PieceCIDV1 string `json:"piece_cid_v1"` + PaddedSize uint64 `json:"padded_size"` + Pieces []storachaBucketRecord `json:"pieces"` +} + +// storachaAggregateCompletion is a per-group commit marker. The aggregate file +// and DB rows may be recoverable after a crash, but this marker is the cheap +// local proof that this exact group was already imported and can be reported to +// the caller. +type storachaAggregateCompletion struct { + GroupID int64 `json:"group_id"` + PieceCID string `json:"piece_cid"` + SubPieces []string `json:"sub_pieces"` + Count int `json:"count"` + PieceCIDV1 string `json:"piece_cid_v1"` + RawSize uint64 `json:"raw_size"` + PaddedSize uint64 `json:"padded_size"` +} + +// storachaAggregateWorkManifest is the durable state machine for a single +// input hash. A stage is marked complete only after its files are fully written, +// synced, renamed where needed, and checksummed. +type storachaAggregateWorkManifest struct { + Version int `json:"version"` + InputSHA256 string `json:"input_sha256"` + InputSize int64 `json:"input_size"` + + RawBuckets storachaAggregateStageManifest `json:"raw_buckets"` + DedupedBuckets storachaAggregateStageManifest `json:"deduped_buckets"` + Groups storachaAggregateGroupManifest `json:"groups"` +} + +// storachaAggregateStageManifest describes append-only JSONL stage artifacts. +// It lets a retry trust a completed stage without rescanning all earlier input. +type storachaAggregateStageManifest struct { + Complete bool `json:"complete"` + Records int64 `json:"records"` + Files []storachaAggregateFileState `json:"files"` +} + +// storachaAggregateGroupManifest records the single grouping artifact. The +// piece count and group count are checked again before processing. +type storachaAggregateGroupManifest struct { + Complete bool `json:"complete"` + Pieces int64 `json:"pieces"` + Groups int64 `json:"groups"` + File storachaAggregateFileState `json:"file"` +} + +// storachaAggregateFileState is deliberately simple metadata: it catches torn +// JSONL records, external edits, and stale artifacts without needing a DB table. +type storachaAggregateFileState struct { + Name string `json:"name"` + Records int64 `json:"records"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +// storachaAggregateBucketCursorState is used only while raw bucketing is +// incomplete. The cursor is written after bucket files and required raw-bucket +// directory entries are synced, so on retry any bytes past these offsets are +// known-uncommitted tails and can be truncated. +type storachaAggregateBucketCursorState struct { + InputSHA256 string `json:"input_sha256"` + InputOffset int64 `json:"input_offset"` + InputLine int64 `json:"input_line"` + Records int64 `json:"records"` + BucketOffsets map[string]int64 `json:"bucket_offsets"` +} + +var aggregatePiecesCmd = &cli.Command{ + Name: "aggregate-pieces", + Usage: "Aggregates already existing pieces from storage into PODSI and park them", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "source", + Usage: "path to read the pieces from", + }, + &cli.StringFlag{ + Name: "target", + Usage: "path to storage directory in Curio's attached permanent storage", + }, + &cli.StringFlag{ + Name: "result", + Usage: "path to write the JSON result", + }, + &cli.StringFlag{ + Name: "input", + Usage: "path to read the CIDs from", + }, + }, + Action: func(cctx *cli.Context) (err error) { + ctx := cctx.Context + var workDir string + + if cctx.String("result") == "" { + return fmt.Errorf("result is required") + } + + resultPath, err := homedir.Expand(cctx.String("result")) + if err != nil { + return xerrors.Errorf("expanding result path: %w", err) + } + resultPath, err = filepath.Abs(filepath.Clean(resultPath)) + if err != nil { + return xerrors.Errorf("resolving result path: %w", err) + } + + defer func() { + // Always write the result file, including errors. This gives the + // caller the list of aggregates that are already committed and a + // machine-readable reason why the run stopped. + writeErr := writeAggregatePiecesResultFromWork(resultPath, workDir, err) + if writeErr != nil { + err = errors.Join(err, writeErr) + } + }() + + if cctx.String("source") == "" || cctx.String("target") == "" || cctx.String("input") == "" { + return fmt.Errorf("source, target and input are required") + } + + inputPath, err := homedir.Expand(cctx.String("input")) + if err != nil { + return xerrors.Errorf("expanding input path: %w", err) + } + inputPath, err = filepath.Abs(filepath.Clean(inputPath)) + if err != nil { + return xerrors.Errorf("expanding input path: %w", err) + } + + // Dedupe is intentionally delegated to the system sort implementation so + // the command can handle tens of millions of PieceCIDv2 records without + // building a process-wide seen set. + sortPath, err := exec.LookPath("sort") + if err != nil { + return xerrors.Errorf("aggregate-pieces requires OS sort in PATH: %w", err) + } + + sourcePath, err := homedir.Expand(cctx.String("source")) + if err != nil { + return xerrors.Errorf("expanding source path: %w", err) + } + sourcePath, err = filepath.Abs(filepath.Clean(sourcePath)) + if err != nil { + return xerrors.Errorf("resolving source path: %w", err) + } + + targetPath, err := homedir.Expand(cctx.String("target")) + if err != nil { + return xerrors.Errorf("expanding target path: %w", err) + } + targetPath, err = filepath.Abs(filepath.Clean(targetPath)) + if err != nil { + return xerrors.Errorf("resolving target path: %w", err) + } + + storageID, err := pieceStorageID(targetPath) + if err != nil { + return xerrors.Errorf("getting storage ID: %w", err) + } + + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + + si := paths.NewDBIndex(curioalerting.NewAlertingSystem(db), db) + + workDir, err = runAggregatePieces(ctx, db, si, storageID, sortPath, inputPath, sourcePath, targetPath) + if err != nil { + return err + } + + return nil + }, +} + +func runAggregatePieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sortPath, inputPath, sourcePath, targetPath string) (string, error) { + // The input hash is part of the workdir path. If the user edits/replaces the + // input, the command starts a new deterministic pipeline instead of trying to + // reuse grouping state that was computed for different contents. + inputSHA256, inputSize, err := hashFileSHA256(inputPath) + if err != nil { + return "", xerrors.Errorf("hashing input file: %w", err) + } + + workDir := filepath.Join(targetPath, storachaAggregateWorkDir, inputSHA256) + if err := os.MkdirAll(workDir, 0755); err != nil { + return workDir, xerrors.Errorf("creating aggregate work directory: %w", err) + } + + manifest, err := loadStorachaAggregateManifest(workDir, inputSHA256, inputSize) + if err != nil { + return workDir, err + } + + // A completed stage is trusted only after its checksum/size/line-count still + // matches the manifest. If any completed artifact changed, the safest answer + // is to stop and require manual cleanup of this hash-specific workdir. + if err := validateCompletedStorachaAggregateStages(workDir, manifest); err != nil { + return workDir, err + } + + // The manifest is monotonic: raw buckets must complete before dedupe, and + // dedupe must complete before groups. The validation above checked the + // deepest completed stage we are resuming from. Each builder below only marks + // its stage complete after durable output is written and recorded, so the + // remaining control flow can stay as a simple forward pipeline. + if !manifest.RawBuckets.Complete { + if err := buildRawStorachaAggregateBuckets(ctx, db, inputPath, inputSHA256, workDir, manifest); err != nil { + return workDir, err + } + } + + if !manifest.DedupedBuckets.Complete { + if err := dedupeStorachaAggregateBuckets(ctx, sortPath, workDir, manifest); err != nil { + return workDir, err + } + } + + if !manifest.Groups.Complete { + if err := buildStorachaAggregateGroups(workDir, manifest); err != nil { + return workDir, err + } + } + + if err := processStorachaAggregateGroups(ctx, db, si, storageID, sourcePath, targetPath, workDir, manifest); err != nil { + return workDir, err + } + + return workDir, nil +} + +func hydrateStorachaSubPieces(ctx context.Context, db *harmonydb.DB, subPieces []pieceDetails) error { + for i := 0; i < len(subPieces); i += storachaAggregateHydrateBatch { + batchEnd := i + storachaAggregateHydrateBatch + if batchEnd > len(subPieces) { + batchEnd = len(subPieces) + } + + batch := subPieces[i:batchEnd] + pcids := make([]string, 0, len(batch)) + psizes := make([]int64, 0, len(batch)) + for _, piece := range batch { + pcids = append(pcids, piece.PieceCID.String()) + psizes = append(psizes, int64(piece.PaddedSize)) + } + + var dbPieces []struct { + Ord int64 `db:"ord"` + ID int64 `db:"id"` + } + // WITH ORDINALITY gives each input tuple a 1-based position inside this + // batch. That ordinal is not a DB identity; it is only the "slot number" + // of the element produced by unnest(). + // + // We need it because the input is allowed to contain duplicates. A plain + // join on piece CID/size would return the same parked_pieces row for each + // duplicate, but without the ordinal we could not safely write the id back + // to the matching slice element. The ordinal keeps hydration positional: + // input slot N gets the first matching complete parked piece for slot N. + err := db.Select(ctx, &dbPieces, ` + WITH input AS ( + SELECT u.piece_cid, u.piece_padded_size, u.ord + FROM unnest($1::text[], $2::bigint[]) WITH ORDINALITY AS u(piece_cid, piece_padded_size, ord) + ) + SELECT i.ord, pp.id + FROM input i + JOIN parked_pieces pp + ON pp.piece_cid = i.piece_cid + AND pp.piece_padded_size = i.piece_padded_size + AND pp.complete = TRUE + AND pp.long_term = TRUE + AND pp.cleanup_task_id IS NULL + ORDER BY i.ord, pp.id`, pcids, psizes) + if err != nil { + return xerrors.Errorf("querying parked_pieces: %w", err) + } + + for _, row := range dbPieces { + if row.Ord < 1 || int(row.Ord) > len(batch) { + return xerrors.Errorf("querying parked_pieces: invalid input ordinal %d", row.Ord) + } + idx := i + int(row.Ord) - 1 + if subPieces[idx].ID == 0 { + subPieces[idx].ID = row.ID + } + } + + for j := i; j < batchEnd; j++ { + if subPieces[j].ID == 0 { + return xerrors.Errorf("piece %s is not a complete active parked piece", subPieces[j].PieceCidV2) + } + } + } + + return nil +} + +func hashFileSHA256(path string) (string, int64, error) { + file, err := os.Open(path) + if err != nil { + return "", 0, xerrors.Errorf("opening %s: %w", path, err) + } + defer func() { + _ = file.Close() + }() + + h := sha256.New() + n, err := io.Copy(h, file) + if err != nil { + return "", 0, xerrors.Errorf("reading %s: %w", path, err) + } + + return hex.EncodeToString(h.Sum(nil)), n, nil +} + +func loadStorachaAggregateManifest(workDir, inputSHA256 string, inputSize int64) (*storachaAggregateWorkManifest, error) { + path := filepath.Join(workDir, storachaAggregateManifestFile) + mb, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return nil, xerrors.Errorf("reading aggregate manifest: %w", err) + } + + manifest := &storachaAggregateWorkManifest{ + Version: storachaAggregateManifestVersion, + InputSHA256: inputSHA256, + InputSize: inputSize, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return nil, err + } + return manifest, nil + } + + var manifest storachaAggregateWorkManifest + if err := json.Unmarshal(mb, &manifest); err != nil { + return nil, xerrors.Errorf("unmarshalling aggregate manifest: %w", err) + } + if manifest.Version != storachaAggregateManifestVersion { + return nil, xerrors.Errorf("aggregate manifest version mismatch: expected %d, got %d; cleanup %s manually before retrying", + storachaAggregateManifestVersion, manifest.Version, workDir) + } + // The workdir name already includes the SHA, but the manifest repeats both + // the SHA and byte length. If either differs, something outside this command + // moved or edited state and automatic recovery would be unsafe. + if manifest.InputSHA256 != inputSHA256 || manifest.InputSize != inputSize { + return nil, xerrors.Errorf("aggregate manifest input mismatch in %s; cleanup manually before retrying", workDir) + } + + return &manifest, nil +} + +func writeStorachaAggregateManifest(workDir string, manifest *storachaAggregateWorkManifest) error { + if err := writeJSONFileAtomic(filepath.Join(workDir, storachaAggregateManifestFile), manifest); err != nil { + return xerrors.Errorf("writing aggregate manifest: %w", err) + } + return nil +} + +func validateCompletedStorachaAggregateStages(workDir string, manifest *storachaAggregateWorkManifest) error { + if manifest.DedupedBuckets.Complete && !manifest.RawBuckets.Complete { + return xerrors.Errorf("aggregate manifest has dedupe complete before raw buckets; cleanup %s manually before retrying", workDir) + } + if manifest.Groups.Complete && !manifest.DedupedBuckets.Complete { + return xerrors.Errorf("aggregate manifest has groups complete before dedupe; cleanup %s manually before retrying", workDir) + } + + // Validate only the deepest completed stage. Each stage is derived entirely + // from the previous one, so once groups.jsonl is complete we only need to + // trust that replay plan; earlier buckets are no longer read. + switch { + case manifest.Groups.Complete: + return validateStorachaAggregateGroupFile(workDir, manifest) + case manifest.DedupedBuckets.Complete: + return validateAggregateFileStates(workDir, manifest.DedupedBuckets.Files) + case manifest.RawBuckets.Complete: + return validateAggregateFileStates(workDir, manifest.RawBuckets.Files) + default: + return nil + } +} + +func validateStorachaAggregateGroupFile(workDir string, manifest *storachaAggregateWorkManifest) error { + if !manifest.Groups.Complete { + return nil + } + + if err := validateAggregateFileState(workDir, manifest.Groups.File); err != nil { + return err + } + if manifest.Groups.File.Records != manifest.Groups.Groups { + return xerrors.Errorf("aggregate group manifest mismatch: file has %d groups, manifest has %d; cleanup %s manually before retrying", + manifest.Groups.File.Records, manifest.Groups.Groups, workDir) + } + + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + file, err := os.Open(groupsPath) + if err != nil { + return xerrors.Errorf("opening aggregate group file: %w", err) + } + defer func() { + _ = file.Close() + }() + + reader := bufio.NewReaderSize(file, 1024*1024) + var groups int64 + var pieces int64 + for { + line, ok, err := readJSONLLine(reader) + if err != nil { + return xerrors.Errorf("reading aggregate group file: %w", err) + } + if !ok { + break + } + + var group storachaAggregateGroup + if err := json.Unmarshal(line, &group); err != nil { + return xerrors.Errorf("decoding aggregate group %d: %w", groups, err) + } + if group.GroupID != groups { + return xerrors.Errorf("aggregate group id mismatch: expected %d, got %d; cleanup %s manually before retrying", groups, group.GroupID, workDir) + } + if err := validateStorachaAggregateGroup(group); err != nil { + return xerrors.Errorf("validating aggregate group %d: %w", group.GroupID, err) + } + + groups++ + pieces += int64(len(group.Pieces)) + } + + if groups != manifest.Groups.Groups { + return xerrors.Errorf("aggregate group count mismatch: file has %d, manifest has %d; cleanup %s manually before retrying", + groups, manifest.Groups.Groups, workDir) + } + if pieces != manifest.Groups.Pieces { + return xerrors.Errorf("aggregate group piece count mismatch: file has %d, manifest has %d; cleanup %s manually before retrying", + pieces, manifest.Groups.Pieces, workDir) + } + + return nil +} + +func validateAggregateFileStates(workDir string, states []storachaAggregateFileState) error { + for _, expected := range states { + if err := validateAggregateFileState(workDir, expected); err != nil { + return err + } + } + return nil +} + +func validateAggregateFileState(workDir string, expected storachaAggregateFileState) error { + actual, err := computeAggregateFileState(workDir, expected.Name) + if err != nil { + return err + } + if actual.Records != expected.Records || actual.Bytes != expected.Bytes || actual.SHA256 != expected.SHA256 { + return xerrors.Errorf("aggregate stage file %s changed; cleanup %s manually before retrying", expected.Name, workDir) + } + return nil +} + +func computeAggregateFileState(workDir, relName string) (storachaAggregateFileState, error) { + path := filepath.Join(workDir, filepath.FromSlash(relName)) + file, err := os.Open(path) + if err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("opening aggregate stage file %s: %w", relName, err) + } + defer func() { + _ = file.Close() + }() + + info, err := file.Stat() + if err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("checking aggregate stage file %s: %w", relName, err) + } + if info.IsDir() { + return storachaAggregateFileState{}, xerrors.Errorf("aggregate stage file is a directory: %s", relName) + } + + h := sha256.New() + reader := bufio.NewReaderSize(file, 1024*1024) + var records int64 + var bytesRead int64 + // Stage artifacts are JSONL and every committed record ends in '\n'. A file + // ending in a partial line is considered torn and is never recorded as a + // completed stage. + for { + line, ok, err := readJSONLLine(reader) + if err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("reading aggregate stage file %s: %w", relName, err) + } + if !ok { + break + } + if _, err := h.Write(line); err != nil { + return storachaAggregateFileState{}, xerrors.Errorf("hashing aggregate stage file %s: %w", relName, err) + } + records++ + bytesRead += int64(len(line)) + } + + if bytesRead != info.Size() { + return storachaAggregateFileState{}, xerrors.Errorf("aggregate stage file %s changed while reading: stat=%d read=%d", relName, info.Size(), bytesRead) + } + + return storachaAggregateFileState{ + Name: filepath.ToSlash(relName), + Records: records, + Bytes: bytesRead, + SHA256: hex.EncodeToString(h.Sum(nil)), + }, nil +} + +func readJSONLLine(reader *bufio.Reader) ([]byte, bool, error) { + line, err := reader.ReadBytes('\n') + if err == nil { + return line, true, nil + } + if errors.Is(err, io.EOF) { + if len(line) == 0 { + return nil, false, nil + } + // EOF after bytes but before '\n' means a writer died mid-record. + // Callers either truncate back to a cursor or reject the completed stage. + return nil, false, xerrors.Errorf("torn JSONL record without trailing newline") + } + return nil, false, err +} + +func buildRawStorachaAggregateBuckets(ctx context.Context, db *harmonydb.DB, inputPath, inputSHA256, workDir string, manifest *storachaAggregateWorkManifest) error { + rawDir := filepath.Join(workDir, storachaAggregateRawBucketDir) + if err := os.MkdirAll(rawDir, 0755); err != nil { + return xerrors.Errorf("creating raw bucket directory: %w", err) + } + + // Raw bucketing is the only non-atomic stage because the input can be huge. + // The cursor gives us a two-file protocol: bucket files are fsynced first, + // newly referenced bucket directory entries are fsynced next, then the cursor + // is atomically replaced with the new input/bucket offsets. On retry we seek + // the input to the cursor and truncate buckets to the last committed offsets + // before appending more records. + cursor, err := loadStorachaAggregateBucketCursor(workDir, inputSHA256) + if err != nil { + return err + } + if err := truncateStorachaAggregateRawBuckets(rawDir, cursor); err != nil { + return err + } + + input, err := os.Open(inputPath) + if err != nil { + return xerrors.Errorf("opening aggregate input: %w", err) + } + defer func() { + _ = input.Close() + }() + + if _, err := input.Seek(cursor.InputOffset, io.SeekStart); err != nil { + return xerrors.Errorf("seeking aggregate input to cursor offset %d: %w", cursor.InputOffset, err) + } + + reader := bufio.NewReaderSize(input, 1024*1024) + batch := make([]pieceDetails, 0, storachaAggregateHydrateBatch) + offset := cursor.InputOffset + lineNumber := cursor.InputLine + records := cursor.Records + + flushBatch := func() error { + if len(batch) == 0 { + return nil + } + // Hydrate only a bounded batch into memory. The durable bucket record + // contains the parked_piece id, both CID forms, raw size, and padded size + // so later stages do not have to ask the DB again for subpiece metadata. + if err := hydrateStorachaSubPieces(ctx, db, batch); err != nil { + return err + } + if err := appendStorachaAggregateRawBucketBatch(rawDir, cursor, batch); err != nil { + return err + } + + // Cursor update is the commit point for this batch. If the process dies + // before this write, the next run truncates any bucket bytes from the + // failed attempt and re-reads these input lines. + records += int64(len(batch)) + cursor.InputOffset = offset + cursor.InputLine = lineNumber + cursor.Records = records + if err := writeStorachaAggregateBucketCursor(workDir, cursor); err != nil { + return err + } + + batch = batch[:0] + return nil + } + + for { + if err := ctx.Err(); err != nil { + return err + } + + line, readErr := reader.ReadBytes('\n') + if len(line) == 0 && errors.Is(readErr, io.EOF) { + break + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return xerrors.Errorf("reading aggregate input line %d: %w", lineNumber+1, readErr) + } + + offset += int64(len(line)) + lineNumber++ + + piece, err := parseStorachaAggregateInputLine(line, lineNumber) + if err != nil { + return err + } + batch = append(batch, piece) + + if len(batch) == storachaAggregateHydrateBatch { + if err := flushBatch(); err != nil { + return err + } + } + if errors.Is(readErr, io.EOF) { + break + } + } + + if err := flushBatch(); err != nil { + return err + } + if records == 0 { + return fmt.Errorf("input has no pieces") + } + + files, err := finishStorachaAggregateRawBuckets(workDir) + if err != nil { + return err + } + if len(files) == 0 { + return xerrors.Errorf("aggregate raw bucketing produced no files") + } + + manifest.RawBuckets = storachaAggregateStageManifest{ + Complete: true, + Records: records, + Files: files, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return err + } + + return nil +} + +func loadStorachaAggregateBucketCursor(workDir, inputSHA256 string) (*storachaAggregateBucketCursorState, error) { + path := filepath.Join(workDir, storachaAggregateBucketCursor) + mb, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return nil, xerrors.Errorf("reading aggregate bucket cursor: %w", err) + } + return &storachaAggregateBucketCursorState{ + InputSHA256: inputSHA256, + BucketOffsets: map[string]int64{}, + }, nil + } + + var cursor storachaAggregateBucketCursorState + if err := json.Unmarshal(mb, &cursor); err != nil { + return nil, xerrors.Errorf("unmarshalling aggregate bucket cursor: %w", err) + } + if cursor.InputSHA256 != inputSHA256 { + return nil, xerrors.Errorf("aggregate bucket cursor input mismatch in %s; cleanup manually before retrying", workDir) + } + if cursor.BucketOffsets == nil { + cursor.BucketOffsets = map[string]int64{} + } + return &cursor, nil +} + +func writeStorachaAggregateBucketCursor(workDir string, cursor *storachaAggregateBucketCursorState) error { + if err := writeJSONFileAtomic(filepath.Join(workDir, storachaAggregateBucketCursor), cursor); err != nil { + return xerrors.Errorf("writing aggregate bucket cursor: %w", err) + } + return nil +} + +func truncateStorachaAggregateRawBuckets(rawDir string, cursor *storachaAggregateBucketCursorState) error { + for _, size := range storachaAggregateBucketSizesAscending() { + name := storachaAggregateBucketFileName(size) + path := filepath.Join(rawDir, name) + offset := cursor.BucketOffsets[name] + + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + if offset != 0 { + return xerrors.Errorf("aggregate raw bucket %s is missing but cursor expects %d bytes; cleanup manually before retrying", path, offset) + } + continue + } + return xerrors.Errorf("checking aggregate raw bucket %s: %w", path, err) + } + if info.IsDir() { + return xerrors.Errorf("aggregate raw bucket is a directory: %s", path) + } + if offset > info.Size() { + return xerrors.Errorf("aggregate raw bucket %s is shorter than cursor offset %d; cleanup manually before retrying", path, offset) + } + if info.Size() != offset { + // The cursor is written only after bucket writes have been flushed and + // synced. Any bytes beyond the cursor are an uncommitted tail from a + // previous crash and must be rolled back before appending more JSONL. + if err := os.Truncate(path, offset); err != nil { + return xerrors.Errorf("truncating aggregate raw bucket %s to cursor offset %d: %w", path, offset, err) + } + } + } + + return nil +} + +func parseStorachaAggregateInputLine(line []byte, lineNumber int64) (pieceDetails, error) { + text := strings.TrimSpace(string(line)) + if text == "" { + return pieceDetails{}, xerrors.Errorf("aggregate input line %d is empty", lineNumber) + } + + pcidV2, err := cid.Parse(text) + if err != nil { + return pieceDetails{}, xerrors.Errorf("parsing aggregate input line %d: %w", lineNumber, err) + } + if !commcidv2.IsPieceCidV2(pcidV2) { + return pieceDetails{}, xerrors.Errorf("aggregate input line %d is not a PieceCIDv2: %s", lineNumber, pcidV2) + } + + // PieceCIDv2 carries the raw payload size. We read raw from the CID, then + // compute padded size from that raw value for parked_pieces lookup. We never + // derive raw size from padded/unpadded size in the aggregate path. + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return pieceDetails{}, xerrors.Errorf("decoding PieceCIDv2 on line %d: %w", lineNumber, err) + } + paddedSize := padreader.PaddedSize(rawSize).Padded() + if err := paddedSize.Validate(); err != nil { + return pieceDetails{}, xerrors.Errorf("invalid padded size for aggregate input line %d: %w", lineNumber, err) + } + if !storachaAggregateIsBucketSize(paddedSize) { + return pieceDetails{}, xerrors.Errorf("aggregate input line %d padded size %d is outside the supported 128-byte to %d-byte bucket range", lineNumber, paddedSize, storachaAggregatePieceSize) + } + + return pieceDetails{ + PieceCidV2: pcidV2, + PieceCID: pcidV1, + RawSize: rawSize, + PaddedSize: paddedSize, + }, nil +} + +type storachaAggregateRawBucketWriter struct { + file *os.File + writer *bufio.Writer + offset int64 + needsRawDirSync bool +} + +func appendStorachaAggregateRawBucketBatch(rawDir string, cursor *storachaAggregateBucketCursorState, batch []pieceDetails) error { + // One batch may touch multiple padded-size buckets. We open only the buckets + // needed for this batch, append JSONL, flush/fsync every touched file, and + // sync the raw bucket directory when a bucket becomes newly referenced. Only + // after those durability steps do we update cursor.BucketOffsets for the + // caller to commit. + writers := map[string]*storachaAggregateRawBucketWriter{} + closeWriters := func() error { + var err error + for _, writer := range writers { + if writer.file != nil { + err = errors.Join(err, writer.file.Close()) + } + } + return err + } + defer func() { + _ = closeWriters() + }() + + openWriter := func(name string) (*storachaAggregateRawBucketWriter, error) { + if writer, ok := writers[name]; ok { + return writer, nil + } + + path := filepath.Join(rawDir, name) + created := false + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0644) + if os.IsNotExist(err) { + file, err = os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY|os.O_APPEND, 0644) + if err == nil { + created = true + } + } + if err != nil { + return nil, xerrors.Errorf("opening aggregate raw bucket %s: %w", path, err) + } + + offset := cursor.BucketOffsets[name] + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, xerrors.Errorf("checking aggregate raw bucket %s: %w", path, err) + } + if info.Size() != offset { + _ = file.Close() + return nil, xerrors.Errorf("aggregate raw bucket %s has size %d, cursor expects %d; cleanup manually before retrying", path, info.Size(), offset) + } + + writer := &storachaAggregateRawBucketWriter{ + file: file, + writer: bufio.NewWriterSize(file, 1024*1024), + offset: offset, + needsRawDirSync: created || offset == 0, + } + writers[name] = writer + return writer, nil + } + + for _, piece := range batch { + record := storachaBucketRecord{ + PieceCIDV2: piece.PieceCidV2.String(), + PieceID: piece.ID, + PieceCIDV1: piece.PieceCID.String(), + RawSize: piece.RawSize, + PaddedSize: uint64(piece.PaddedSize), + } + + name := storachaAggregateBucketFileName(piece.PaddedSize) + writer, err := openWriter(name) + if err != nil { + return err + } + n, err := writeJSONLine(writer.writer, record) + if err != nil { + return xerrors.Errorf("writing aggregate raw bucket %s: %w", name, err) + } + writer.offset += int64(n) + } + + newOffsets := map[string]int64{} + needsRawDirSync := false + for name, writer := range writers { + if err := writer.writer.Flush(); err != nil { + return xerrors.Errorf("flushing aggregate raw bucket %s: %w", name, err) + } + if err := writer.file.Sync(); err != nil { + return xerrors.Errorf("syncing aggregate raw bucket %s: %w", name, err) + } + if err := writer.file.Close(); err != nil { + return xerrors.Errorf("closing aggregate raw bucket %s: %w", name, err) + } + writer.file = nil + newOffsets[name] = writer.offset + needsRawDirSync = needsRawDirSync || writer.needsRawDirSync + } + + if needsRawDirSync { + if err := syncDir(rawDir); err != nil { + return xerrors.Errorf("syncing aggregate raw bucket directory before cursor commit: %w", err) + } + } + + for name, offset := range newOffsets { + cursor.BucketOffsets[name] = offset + } + + return nil +} + +func finishStorachaAggregateRawBuckets(workDir string) ([]storachaAggregateFileState, error) { + rawDir := filepath.Join(workDir, storachaAggregateRawBucketDir) + var files []storachaAggregateFileState + var removedEmpty bool + + // Some size buckets may be opened but receive no records in small test runs + // or interrupted runs. Delete empties before recording the stage so later + // stages only scan files that contain data. + for _, size := range storachaAggregateBucketSizesAscending() { + rel := filepath.ToSlash(filepath.Join(storachaAggregateRawBucketDir, storachaAggregateBucketFileName(size))) + path := filepath.Join(workDir, filepath.FromSlash(rel)) + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, xerrors.Errorf("checking aggregate raw bucket %s: %w", path, err) + } + if info.IsDir() { + return nil, xerrors.Errorf("aggregate raw bucket is a directory: %s", path) + } + if info.Size() == 0 { + if err := os.Remove(path); err != nil { + return nil, xerrors.Errorf("removing empty aggregate raw bucket %s: %w", path, err) + } + removedEmpty = true + continue + } + + state, err := computeAggregateFileState(workDir, rel) + if err != nil { + return nil, err + } + files = append(files, state) + } + + if removedEmpty { + if err := syncDir(rawDir); err != nil { + return nil, err + } + } + + return files, nil +} + +func dedupeStorachaAggregateBuckets(ctx context.Context, sortPath, workDir string, manifest *storachaAggregateWorkManifest) error { + if !manifest.RawBuckets.Complete { + return xerrors.Errorf("aggregate raw buckets are not complete") + } + + dedupeDir := filepath.Join(workDir, storachaAggregateDedupeDir) + if err := os.MkdirAll(dedupeDir, 0755); err != nil { + return xerrors.Errorf("creating dedupe bucket directory: %w", err) + } + + // Dedupe is per padded-size bucket. Because PieceCIDv2 is the first JSON + // field, byte sorting the whole line groups duplicate input records together. + // This keeps memory bounded even when the input contains tens of millions of + // pieces. + var files []storachaAggregateFileState + var totalRecords int64 + for _, rawState := range manifest.RawBuckets.Files { + if err := ctx.Err(); err != nil { + return err + } + + rawPath := filepath.Join(workDir, filepath.FromSlash(rawState.Name)) + sortedTmp := filepath.Join(dedupeDir, filepath.Base(rawState.Name)+".sorted.tmp") + + cmd := exec.CommandContext(ctx, sortPath, "-o", sortedTmp, rawPath) + cmd.Env = append(os.Environ(), "LC_ALL=C") + sortOut, err := cmd.CombinedOutput() + if err != nil { + return xerrors.Errorf("sorting aggregate bucket %s: %w: %s", rawState.Name, err, strings.TrimSpace(string(sortOut))) + } + + finalRel := filepath.ToSlash(filepath.Join(storachaAggregateDedupeDir, filepath.Base(rawState.Name))) + finalPath := filepath.Join(workDir, filepath.FromSlash(finalRel)) + records, err := dedupeSortedStorachaAggregateBucket(sortedTmp, finalPath) + removeErr := os.Remove(sortedTmp) + if removeErr != nil && !os.IsNotExist(removeErr) { + err = errors.Join(err, xerrors.Errorf("removing sorted aggregate bucket temp %s: %w", sortedTmp, removeErr)) + } + if err != nil { + return err + } + if records == 0 { + if removeErr := os.Remove(finalPath); removeErr != nil && !os.IsNotExist(removeErr) { + return xerrors.Errorf("removing empty deduped aggregate bucket %s: %w", finalPath, removeErr) + } + continue + } + + state, err := computeAggregateFileState(workDir, finalRel) + if err != nil { + return err + } + if state.Records != records { + return xerrors.Errorf("deduped aggregate bucket %s record count changed while finalizing", finalRel) + } + files = append(files, state) + totalRecords += records + } + + if totalRecords == 0 { + return xerrors.Errorf("aggregate dedupe produced no records") + } + + manifest.DedupedBuckets = storachaAggregateStageManifest{ + Complete: true, + Records: totalRecords, + Files: files, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return err + } + + return nil +} + +func dedupeSortedStorachaAggregateBucket(sortedPath, finalPath string) (int64, error) { + input, err := os.Open(sortedPath) + if err != nil { + return 0, xerrors.Errorf("opening sorted aggregate bucket %s: %w", sortedPath, err) + } + defer func() { + _ = input.Close() + }() + + tmpPath := finalPath + ".tmp" + output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return 0, xerrors.Errorf("creating deduped aggregate bucket %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + reader := bufio.NewReaderSize(input, 1024*1024) + writer := bufio.NewWriterSize(output, 1024*1024) + var previousPieceCIDV2 string + var records int64 + // The sorted input makes duplicates adjacent. Keep the first hydrated record + // and drop later records for the same PieceCIDv2; all retained records are + // written back as canonical JSONL with a newline terminator. + for { + line, ok, err := readJSONLLine(reader) + if err != nil { + _ = output.Close() + return 0, xerrors.Errorf("reading sorted aggregate bucket %s: %w", sortedPath, err) + } + if !ok { + break + } + + var record storachaBucketRecord + if err := json.Unmarshal(line, &record); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("decoding sorted aggregate bucket %s: %w", sortedPath, err) + } + if err := validateStorachaBucketRecord(record); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("validating sorted aggregate bucket %s: %w", sortedPath, err) + } + if record.PieceCIDV2 == previousPieceCIDV2 { + continue + } + + if _, err := writeJSONLine(writer, record); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("writing deduped aggregate bucket %s: %w", tmpPath, err) + } + previousPieceCIDV2 = record.PieceCIDV2 + records++ + } + + if err := writer.Flush(); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("flushing deduped aggregate bucket %s: %w", tmpPath, err) + } + if err := output.Sync(); err != nil { + _ = output.Close() + return 0, xerrors.Errorf("syncing deduped aggregate bucket %s: %w", tmpPath, err) + } + if err := output.Close(); err != nil { + return 0, xerrors.Errorf("closing deduped aggregate bucket %s: %w", tmpPath, err) + } + + if records == 0 { + removeTmp = false + if err := os.Remove(tmpPath); err != nil { + return 0, xerrors.Errorf("removing empty deduped aggregate bucket %s: %w", tmpPath, err) + } + return 0, nil + } + + if err := os.Rename(tmpPath, finalPath); err != nil { + return 0, xerrors.Errorf("renaming deduped aggregate bucket %s: %w", finalPath, err) + } + removeTmp = false + if err := syncDir(filepath.Dir(finalPath)); err != nil { + return 0, err + } + + return records, nil +} + +type storachaAggregateBucketReader struct { + // pending is the next unconsumed record from this size bucket. Keeping one + // record buffered per bucket lets grouping stream all buckets without loading + // bucket files into memory. + size abi.PaddedPieceSize + path string + file *os.File + reader *bufio.Reader + pending *storachaBucketRecord +} + +func buildStorachaAggregateGroups(workDir string, manifest *storachaAggregateWorkManifest) error { + if !manifest.DedupedBuckets.Complete { + return xerrors.Errorf("aggregate dedupe buckets are not complete") + } + + // Grouping reads from at most one file for each power-of-two padded size. + // For the 128-byte through 1-GiB range that is 24 readers, independent of + // how many total pieces are in the migration input. + readers, err := openStorachaAggregateBucketReaders(workDir, manifest.DedupedBuckets.Files) + if err != nil { + return err + } + defer func() { + for _, reader := range readers { + _ = reader.close() + } + }() + + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + tmpPath := groupsPath + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating aggregate group temp file: %w", err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + writer := bufio.NewWriterSize(file, 1024*1024) + maxEntries := datasegment.MaxIndexEntriesInDeal(storachaAggregatePieceSize) + var groupID int64 + var groupedPieces int64 + for groupedPieces < manifest.DedupedBuckets.Records { + placement := storachaAggregatePlacement{} + groupPieces := make([]storachaBucketRecord, 0, int(maxEntries)) + + // The greedy walk restarts from the largest bucket for every aggregate. + // Within a size bucket we keep sorted input order. We consume as many + // same-size pieces as fit, then move exactly one power smaller. With + // power-of-two piece sizes this keeps placement dense and deterministic. + for _, bucket := range readers { + for bucket.pending != nil { + nextOffsetNodes, fits, err := placement.tryAdd(abi.PaddedPieceSize(bucket.pending.PaddedSize), maxEntries) + if err != nil { + return xerrors.Errorf("checking aggregate fit for group %d: %w", groupID, err) + } + if !fits { + if len(groupPieces) == 0 { + return xerrors.Errorf("piece %s with padded size %d does not fit in a %d padded aggregate after reserving the datasegment index", + bucket.pending.PieceCIDV2, bucket.pending.PaddedSize, storachaAggregatePieceSize) + } + break + } + + groupPieces = append(groupPieces, *bucket.pending) + placement.accept(nextOffsetNodes) + if err := bucket.advance(); err != nil { + return err + } + } + } + + if len(groupPieces) == 0 { + return xerrors.Errorf("next aggregate subpiece does not fit in a %d padded aggregate after reserving the datasegment index", storachaAggregatePieceSize) + } + + // The fit check above is intentionally cheap. Before we persist the + // group, datasegment builds the aggregate metadata and computes the exact + // PieceCIDv1 that this replay plan must later produce. + expectedCIDV1, err := expectedStorachaAggregatePieceCID(groupPieces) + if err != nil { + return xerrors.Errorf("validating aggregate group %d with datasegment: %w", groupID, err) + } + + group := storachaAggregateGroup{ + GroupID: groupID, + PieceCIDV1: expectedCIDV1.String(), + PaddedSize: uint64(storachaAggregatePieceSize), + Pieces: groupPieces, + } + if _, err := writeJSONLine(writer, group); err != nil { + return xerrors.Errorf("writing aggregate group %d: %w", groupID, err) + } + + groupID++ + groupedPieces += int64(len(groupPieces)) + } + + if groupedPieces != manifest.DedupedBuckets.Records { + return xerrors.Errorf("aggregate grouping mismatch: grouped %d pieces, expected %d", groupedPieces, manifest.DedupedBuckets.Records) + } + + if err := writer.Flush(); err != nil { + _ = file.Close() + return xerrors.Errorf("flushing aggregate group file: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return xerrors.Errorf("syncing aggregate group file: %w", err) + } + if err := file.Close(); err != nil { + return xerrors.Errorf("closing aggregate group file: %w", err) + } + if err := os.Rename(tmpPath, groupsPath); err != nil { + return xerrors.Errorf("renaming aggregate group file: %w", err) + } + removeTmp = false + if err := syncDir(workDir); err != nil { + return err + } + + state, err := computeAggregateFileState(workDir, storachaAggregateGroupsFile) + if err != nil { + return err + } + manifest.Groups = storachaAggregateGroupManifest{ + Complete: true, + Pieces: groupedPieces, + Groups: groupID, + File: state, + } + if err := writeStorachaAggregateManifest(workDir, manifest); err != nil { + return err + } + + return nil +} + +func openStorachaAggregateBucketReaders(workDir string, files []storachaAggregateFileState) ([]*storachaAggregateBucketReader, error) { + filesByBucket := map[string]storachaAggregateFileState{} + for _, state := range files { + filesByBucket[filepath.Base(state.Name)] = state + } + + var readers []*storachaAggregateBucketReader + // Readers are opened largest first so the grouping pass starts every + // aggregate with the largest available pieces, then walks one power smaller + // when the current size no longer fits. + for _, size := range storachaAggregateBucketSizesDescending() { + name := storachaAggregateBucketFileName(size) + state, ok := filesByBucket[name] + if !ok { + continue + } + + path := filepath.Join(workDir, filepath.FromSlash(state.Name)) + file, err := os.Open(path) + if err != nil { + for _, reader := range readers { + _ = reader.close() + } + return nil, xerrors.Errorf("opening deduped aggregate bucket %s: %w", state.Name, err) + } + + reader := &storachaAggregateBucketReader{ + size: size, + path: path, + file: file, + reader: bufio.NewReaderSize(file, 1024*1024), + } + if err := reader.advance(); err != nil { + _ = reader.close() + for _, existing := range readers { + _ = existing.close() + } + return nil, err + } + readers = append(readers, reader) + } + + return readers, nil +} + +func (r *storachaAggregateBucketReader) advance() error { + line, ok, err := readJSONLLine(r.reader) + if err != nil { + return xerrors.Errorf("reading deduped aggregate bucket %s: %w", r.path, err) + } + if !ok { + r.pending = nil + return nil + } + + var record storachaBucketRecord + if err := json.Unmarshal(line, &record); err != nil { + return xerrors.Errorf("decoding deduped aggregate bucket %s: %w", r.path, err) + } + if err := validateStorachaBucketRecord(record); err != nil { + return xerrors.Errorf("validating deduped aggregate bucket %s: %w", r.path, err) + } + if record.PaddedSize != uint64(r.size) { + return xerrors.Errorf("deduped aggregate bucket %s contains padded size %d, expected %d", r.path, record.PaddedSize, r.size) + } + + r.pending = &record + return nil +} + +func (r *storachaAggregateBucketReader) close() error { + if r.file == nil { + return nil + } + err := r.file.Close() + r.file = nil + return err +} + +type storachaAggregatePlacement struct { + nextOffsetNodes uint64 + count uint +} + +func (p storachaAggregatePlacement) tryAdd(size abi.PaddedPieceSize, maxEntries uint) (uint64, bool, error) { + if err := size.Validate(); err != nil { + return 0, false, err + } + if p.count >= maxEntries { + return 0, false, nil + } + + // This is the same alignment step used by datasegment.ComputeDealPlacement: + // choose the next location aligned for this piece size, then advance by one + // piece. We use it only to decide whether a candidate can fit cheaply while + // streaming bucket files. Before any group is committed, NewAggregate below + // recomputes placement and is the authoritative validation. + sizeInNodes := uint64(size) / 32 + index := (p.nextOffsetNodes + sizeInNodes - 1) / sizeInNodes + nextOffsetNodes := (index + 1) * sizeInNodes + packedPaddedSize := nextOffsetNodes * 32 + indexPaddedSize := uint64(maxEntries) * datasegment.EntrySize + + return nextOffsetNodes, packedPaddedSize+indexPaddedSize <= uint64(storachaAggregatePieceSize), nil +} + +func (p *storachaAggregatePlacement) accept(nextOffsetNodes uint64) { + p.nextOffsetNodes = nextOffsetNodes + p.count++ +} + +func validateStorachaAggregateGroup(group storachaAggregateGroup) error { + if group.PaddedSize != uint64(storachaAggregatePieceSize) { + return xerrors.Errorf("aggregate group padded size is %d, expected %d", group.PaddedSize, storachaAggregatePieceSize) + } + if len(group.Pieces) == 0 { + return xerrors.Errorf("aggregate group has no pieces") + } + for i, piece := range group.Pieces { + if err := validateStorachaBucketRecord(piece); err != nil { + return xerrors.Errorf("piece %d: %w", i, err) + } + } + + // Completed grouping state is trusted only if datasegment still agrees with + // the exact ordered list of subpieces and the precomputed aggregate CID. + expectedCIDV1, err := expectedStorachaAggregatePieceCID(group.Pieces) + if err != nil { + return err + } + if group.PieceCIDV1 != expectedCIDV1.String() { + return xerrors.Errorf("aggregate group commP mismatch: group has %s, datasegment computes %s", group.PieceCIDV1, expectedCIDV1) + } + + return nil +} + +func expectedStorachaAggregatePieceCID(records []storachaBucketRecord) (cid.Cid, error) { + // This computes the expected aggregate PieceCIDv1 from the subpiece CIDs and + // padded sizes only. The PieceCIDv2 for the aggregate is not known until the + // aggregate body is actually written, because v2 also carries raw bytes. + pinfos, err := storachaBucketRecordsToPieceInfos(records) + if err != nil { + return cid.Undef, err + } + aggr, err := datasegment.NewAggregate(storachaAggregatePieceSize, pinfos) + if err != nil { + return cid.Undef, err + } + pcid, err := aggr.PieceCID() + if err != nil { + return cid.Undef, xerrors.Errorf("computing aggregate PieceCIDv1: %w", err) + } + return pcid, nil +} + +func storachaBucketRecordsToPieceInfos(records []storachaBucketRecord) ([]abi.PieceInfo, error) { + pinfos := make([]abi.PieceInfo, 0, len(records)) + for i, record := range records { + pcidV1, err := cid.Parse(record.PieceCIDV1) + if err != nil { + return nil, xerrors.Errorf("parsing PieceCIDv1 for record %d: %w", i, err) + } + pinfos = append(pinfos, abi.PieceInfo{ + PieceCID: pcidV1, + Size: abi.PaddedPieceSize(record.PaddedSize), + }) + } + return pinfos, nil +} + +func validateStorachaBucketRecord(record storachaBucketRecord) error { + if record.PieceID <= 0 { + return xerrors.Errorf("invalid parked piece id %d", record.PieceID) + } + + // A bucket record is self-validating: PieceCIDv2 gives us PieceCIDv1 and raw + // size, raw size gives us padded size, and the bucket filename/checking code + // ensures the record lives in the expected power-of-two bucket. + pcidV2, err := cid.Parse(record.PieceCIDV2) + if err != nil { + return xerrors.Errorf("parsing PieceCIDv2 %q: %w", record.PieceCIDV2, err) + } + if !commcidv2.IsPieceCidV2(pcidV2) { + return xerrors.Errorf("not a PieceCIDv2: %s", pcidV2) + } + + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return xerrors.Errorf("decoding PieceCIDv2 %s: %w", pcidV2, err) + } + if pcidV1.String() != record.PieceCIDV1 { + return xerrors.Errorf("PieceCIDv1 mismatch for %s: record has %s, CID decodes to %s", pcidV2, record.PieceCIDV1, pcidV1) + } + if rawSize != record.RawSize { + return xerrors.Errorf("raw size mismatch for %s: record has %d, CID decodes to %d", pcidV2, record.RawSize, rawSize) + } + paddedSize := padreader.PaddedSize(rawSize).Padded() + if uint64(paddedSize) != record.PaddedSize { + return xerrors.Errorf("padded size mismatch for %s: record has %d, raw size pads to %d", pcidV2, record.PaddedSize, paddedSize) + } + if err := paddedSize.Validate(); err != nil { + return xerrors.Errorf("invalid padded size for %s: %w", pcidV2, err) + } + if !storachaAggregateIsBucketSize(paddedSize) { + return xerrors.Errorf("padded size %d is outside supported aggregate bucket range", paddedSize) + } + + return nil +} + +func processStorachaAggregateGroups(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath, workDir string, manifest *storachaAggregateWorkManifest) error { + for _, dir := range []string{ + filepath.Join(workDir, storachaAggregateCompleteDir), + filepath.Join(workDir, storachaAggregateTempDir), + storachaAggregateStagingDir(targetPath), + storachaAggregateStagedDir(targetPath), + } { + if err := os.MkdirAll(dir, 0755); err != nil { + return xerrors.Errorf("creating aggregate directory %s: %w", dir, err) + } + } + + // Processing is a linear replay of groups.jsonl. Group ids must match the + // file order so completion marker names are deterministic and partial result + // reporting can stream the same plan. + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + file, err := os.Open(groupsPath) + if err != nil { + return xerrors.Errorf("opening aggregate group file: %w", err) + } + defer func() { + _ = file.Close() + }() + + reader := bufio.NewReaderSize(file, 1024*1024) + var groups int64 + for { + if err := ctx.Err(); err != nil { + return err + } + + line, ok, err := readJSONLLine(reader) + if err != nil { + return xerrors.Errorf("reading aggregate group file: %w", err) + } + if !ok { + break + } + + var group storachaAggregateGroup + if err := json.Unmarshal(line, &group); err != nil { + return xerrors.Errorf("decoding aggregate group %d: %w", groups, err) + } + if group.GroupID != groups { + return xerrors.Errorf("aggregate group id mismatch while processing: expected %d, got %d", groups, group.GroupID) + } + if err := processStorachaAggregateGroup(ctx, db, si, storageID, sourcePath, targetPath, workDir, group); err != nil { + return err + } + groups++ + } + + if groups != manifest.Groups.Groups { + return xerrors.Errorf("processed %d aggregate groups, manifest expects %d; cleanup %s manually before retrying", groups, manifest.Groups.Groups, workDir) + } + + return nil +} + +func processStorachaAggregateGroup(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath, workDir string, group storachaAggregateGroup) error { + // Recovery order is from most committed to least committed: + // 1. completion marker: nothing else to do; + // 2. final Curio piece file + DB row: recreate metadata; + // 3. committed staging marker + staging file: run the normal import path; + // 4. no committed file: rebuild from read-only source pieces. + if _, ok, err := readCompletedStorachaAggregateGroup(workDir, group); err != nil { + return err + } else if ok { + return nil + } + + expectedCIDV1, err := cid.Parse(group.PieceCIDV1) + if err != nil { + return xerrors.Errorf("parsing aggregate group %d PieceCIDv1: %w", group.GroupID, err) + } + expectedPaddedSize := abi.PaddedPieceSize(group.PaddedSize) + + if recovered, err := recoverStorachaAggregateFromFinal(ctx, db, si, storageID, targetPath, workDir, group, expectedCIDV1, expectedPaddedSize); err != nil { + return err + } else if recovered { + return nil + } + + if recovered, err := recoverStorachaAggregateFromStaging(ctx, db, si, storageID, targetPath, workDir, group, expectedCIDV1, expectedPaddedSize); err != nil { + return err + } else if recovered { + return nil + } + + tempPath, verified, err := createStorachaAggregateTempFile(sourcePath, workDir, group, expectedCIDV1, expectedPaddedSize) + if err != nil { + return err + } + stagingPath, err := stageStorachaAggregateFile(targetPath, group, tempPath, verified) + if err != nil { + return err + } + return importStagedStorachaAggregateFile(ctx, db, si, storageID, targetPath, workDir, group, stagingPath, verified) +} + +func readCompletedStorachaAggregateGroup(workDir string, group storachaAggregateGroup) (storachaAggregateCompletion, bool, error) { + path := storachaAggregateCompletionPath(workDir, group.GroupID) + mb, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return storachaAggregateCompletion{}, false, nil + } + return storachaAggregateCompletion{}, false, xerrors.Errorf("reading aggregate completion marker %s: %w", path, err) + } + + var complete storachaAggregateCompletion + if err := json.Unmarshal(mb, &complete); err != nil { + return storachaAggregateCompletion{}, false, xerrors.Errorf("decoding aggregate completion marker %s: %w; cleanup manually before retrying", path, err) + } + // A marker is trusted only if it describes this exact group. A corrupt or + // stale marker would otherwise make us skip a group that still needs work. + if err := validateStorachaAggregateCompletion(group, complete); err != nil { + return storachaAggregateCompletion{}, false, xerrors.Errorf("invalid aggregate completion marker %s: %w; cleanup manually before retrying", path, err) + } + + return complete, true, nil +} + +func validateStorachaAggregateCompletion(group storachaAggregateGroup, complete storachaAggregateCompletion) error { + if complete.GroupID != group.GroupID { + return xerrors.Errorf("group id mismatch: marker has %d, group has %d", complete.GroupID, group.GroupID) + } + if complete.PieceCIDV1 != group.PieceCIDV1 { + return xerrors.Errorf("PieceCIDv1 mismatch: marker has %s, group has %s", complete.PieceCIDV1, group.PieceCIDV1) + } + if complete.PaddedSize != group.PaddedSize { + return xerrors.Errorf("padded size mismatch: marker has %d, group has %d", complete.PaddedSize, group.PaddedSize) + } + if complete.Count != len(group.Pieces) || len(complete.SubPieces) != len(group.Pieces) { + return xerrors.Errorf("subpiece count mismatch: marker has %d/%d, group has %d", complete.Count, len(complete.SubPieces), len(group.Pieces)) + } + pcidV2, err := cid.Parse(complete.PieceCID) + if err != nil { + return xerrors.Errorf("parsing aggregate PieceCIDv2 %q: %w", complete.PieceCID, err) + } + if !commcidv2.IsPieceCidV2(pcidV2) { + return xerrors.Errorf("aggregate PieceCID is not v2: %s", pcidV2) + } + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return xerrors.Errorf("decoding aggregate PieceCIDv2 %s: %w", pcidV2, err) + } + if pcidV1.String() != complete.PieceCIDV1 { + return xerrors.Errorf("aggregate PieceCIDv1 mismatch: marker has %s, v2 decodes to %s", complete.PieceCIDV1, pcidV1) + } + if rawSize != complete.RawSize { + return xerrors.Errorf("aggregate raw size mismatch: marker has %d, v2 decodes to %d", complete.RawSize, rawSize) + } + if uint64(padreader.PaddedSize(rawSize).Padded()) != complete.PaddedSize { + return xerrors.Errorf("aggregate padded size mismatch: marker has %d, raw size pads to %d", complete.PaddedSize, padreader.PaddedSize(rawSize).Padded()) + } + for i, piece := range group.Pieces { + if complete.SubPieces[i] != piece.PieceCIDV2 { + return xerrors.Errorf("subpiece %d mismatch: marker has %s, group has %s", i, complete.SubPieces[i], piece.PieceCIDV2) + } + } + return nil +} + +func recoverStorachaAggregateFromFinal(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, workDir string, group storachaAggregateGroup, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (bool, error) { + // A crash can happen after import-pieces moved the file into Curio storage + // but before aggregate-pieces wrote its completion marker. At this point the + // file has passed the staging marker boundary, so we trust the DB row and final + // filename instead of re-reading the whole aggregate for CommP. + var rows []storachaFinalRecoveryRow + err := db.Select(ctx, &rows, ` + SELECT id, piece_cid, piece_padded_size, piece_raw_size + FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND long_term = TRUE + AND cleanup_task_id IS NULL + ORDER BY id`, expectedCIDV1.String(), int64(expectedPaddedSize)) + if err != nil { + return false, xerrors.Errorf("querying aggregate parked pieces: %w", err) + } + + for _, row := range rows { + finalPath := storachaFinalPiecePath(targetPath, row.ID) + info, err := os.Stat(finalPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return false, xerrors.Errorf("checking aggregate final file %s: %w", finalPath, err) + } + if info.IsDir() { + return false, xerrors.Errorf("aggregate final path is a directory: %s", finalPath) + } + if info.Size() < 0 || uint64(info.Size()) != uint64(row.RawSize) { + return false, xerrors.Errorf("aggregate final file %s size mismatch for parked piece %d: db=%d file=%d; cleanup manually before retrying", finalPath, row.ID, row.RawSize, info.Size()) + } + + verified, err := storachaVerifiedAggregatePieceFromRow(row) + if err != nil { + return false, err + } + + imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row, false) + if err != nil { + return false, err + } + if !imported { + return false, nil + } + + if err := finishStorachaAggregateGroup(targetPath, workDir, group, verified); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +func recoverStorachaAggregateFromStaging(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, workDir string, group storachaAggregateGroup, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (bool, error) { + // Staged markers are the commit point for staging. The marker filename is the + // aggregate PieceCIDv2; if it decodes to this group, the matching .car has + // already been written, fsynced, named from CommP, and committed by marker. + staged := storachaAggregateStagedDir(targetPath) + entries, err := os.ReadDir(staged) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, xerrors.Errorf("reading aggregate staged marker directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + return false, xerrors.Errorf("aggregate staged marker is a directory: %s; cleanup manually before retrying", filepath.Join(staged, entry.Name())) + } + + pcidV2, err := cid.Parse(entry.Name()) + if err != nil || !commcidv2.IsPieceCidV2(pcidV2) { + return false, xerrors.Errorf("aggregate staged marker is not a PieceCIDv2: %s; cleanup manually before retrying", filepath.Join(staged, entry.Name())) + } + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + if err != nil { + return false, xerrors.Errorf("decoding aggregate staged marker %s: %w; cleanup manually before retrying", filepath.Join(staged, entry.Name()), err) + } + if !pcidV1.Equals(expectedCIDV1) || padreader.PaddedSize(rawSize).Padded() != expectedPaddedSize { + continue + } + + stagingPath := storachaAggregateStagingPath(targetPath, pcidV2) + info, err := os.Stat(stagingPath) + if err != nil { + if os.IsNotExist(err) { + return false, xerrors.Errorf("aggregate staged marker %s exists but staging file %s is missing; cleanup manually before retrying", filepath.Join(staged, entry.Name()), stagingPath) + } + return false, xerrors.Errorf("checking aggregate staging file %s: %w", stagingPath, err) + } + if info.IsDir() { + return false, xerrors.Errorf("aggregate staging path is a directory: %s", stagingPath) + } + if info.Size() < 0 || uint64(info.Size()) != rawSize { + return false, xerrors.Errorf("aggregate staging file %s size mismatch: marker raw=%d file=%d; cleanup manually before retrying", stagingPath, rawSize, info.Size()) + } + + verified := storachaVerifiedAggregatePiece{ + PieceCIDV2: pcidV2, + PieceCIDV1: pcidV1, + RawSize: rawSize, + PaddedSize: expectedPaddedSize, + } + + if err := importStagedStorachaAggregateFile(ctx, db, si, storageID, targetPath, workDir, group, stagingPath, verified); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +type storachaVerifiedAggregatePiece struct { + // RawSize is the exact number of bytes written/read for the aggregate file. + // PaddedSize and PieceCIDv1 come from CommP. PieceCIDv2 is computed from the + // same digest plus RawSize; there is no padded/unpadded-to-raw conversion. + PieceCIDV2 cid.Cid + PieceCIDV1 cid.Cid + RawSize uint64 + PaddedSize abi.PaddedPieceSize +} + +func storachaVerifiedAggregatePieceFromRow(row storachaFinalRecoveryRow) (storachaVerifiedAggregatePiece, error) { + pcidV1, err := cid.Parse(row.PieceCID) + if err != nil { + return storachaVerifiedAggregatePiece{}, xerrors.Errorf("parsing aggregate final PieceCIDv1 %s: %w", row.PieceCID, err) + } + + pcidV2, err := commcid.PieceCidV2FromV1(pcidV1, uint64(row.RawSize)) + if err != nil { + return storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate final PieceCIDv2 from DB row: %w", err) + } + + return storachaVerifiedAggregatePiece{ + PieceCIDV2: pcidV2, + PieceCIDV1: pcidV1, + RawSize: uint64(row.RawSize), + PaddedSize: abi.PaddedPieceSize(row.PaddedSize), + }, nil +} + +func createStorachaAggregateTempFile(sourcePath, workDir string, group storachaAggregateGroup, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (string, storachaVerifiedAggregatePiece, error) { + // Rebuild the datasegment aggregate from the deterministic group file, then + // stream read-only source/piece/s-t00- files into a local + // temp aggregate. The temp file is fsynced and handed to staging only after + // its CommP and padded size match the group plan. + pinfos, err := storachaBucketRecordsToPieceInfos(group.Pieces) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, err + } + + aggr, err := datasegment.NewAggregate(expectedPaddedSize, pinfos) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate group %d: %w", group.GroupID, err) + } + pcidV1, err := aggr.PieceCID() + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d PieceCIDv1: %w", group.GroupID, err) + } + if !pcidV1.Equals(expectedCIDV1) { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("aggregate group %d PieceCIDv1 changed: datasegment computes %s, group has %s", group.GroupID, pcidV1, expectedCIDV1) + } + + readers, closeReaders, err := openStorachaAggregateSubPieceReaders(sourcePath, group.Pieces) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, err + } + defer func() { + _ = closeReaders() + }() + + outR, err := aggr.AggregateObjectReader(readers) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate reader for group %d: %w", group.GroupID, err) + } + + tmpPath := storachaAggregateTempFilePath(workDir, group.GroupID) + if err := os.MkdirAll(filepath.Dir(tmpPath), 0755); err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate temp directory: %w", err) + } + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("creating aggregate temp file %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + cp := new(commp.Calc) + defer cp.Reset() + n, copyErr := io.Copy(io.MultiWriter(cp, file), outR) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(copyErr, syncErr, closeErr); err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("writing aggregate group %d: %w", group.GroupID, err) + } + + digest, actualPaddedSize, err := cp.Digest() + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d digest: %w", group.GroupID, err) + } + if abi.PaddedPieceSize(actualPaddedSize) != expectedPaddedSize { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("aggregate group %d padded size mismatch: expected %d, got %d", group.GroupID, expectedPaddedSize, actualPaddedSize) + } + + actualCIDV1, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d PieceCIDv1 from digest: %w", group.GroupID, err) + } + if !actualCIDV1.Equals(expectedCIDV1) { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("aggregate group %d commP mismatch: calculated %s, expected %s", group.GroupID, actualCIDV1, expectedCIDV1) + } + + // Raw size is the number of bytes AggregateObjectReader actually emitted. + // It is not inferred from padded size. This value is what PieceCIDv2 needs. + actualCIDV2, err := commcid.DataCommitmentToPieceCidv2(digest, uint64(n)) + if err != nil { + return "", storachaVerifiedAggregatePiece{}, xerrors.Errorf("computing aggregate group %d PieceCIDv2: %w", group.GroupID, err) + } + + removeTmp = false + + return tmpPath, storachaVerifiedAggregatePiece{ + PieceCIDV2: actualCIDV2, + PieceCIDV1: actualCIDV1, + RawSize: uint64(n), + PaddedSize: abi.PaddedPieceSize(actualPaddedSize), + }, nil +} + +func stageStorachaAggregateFile(targetPath string, group storachaAggregateGroup, tempPath string, verified storachaVerifiedAggregatePiece) (string, error) { + // This is the only place that turns a verified aggregate temp file into + // committed staging state. The order is: rename to .car, fsync + // directories, then write staged/. Staging recovery trusts only + // files that have crossed this marker boundary. + staging := storachaAggregateStagingDir(targetPath) + if err := os.MkdirAll(staging, 0755); err != nil { + return "", xerrors.Errorf("creating aggregate staging directory: %w", err) + } + if err := os.MkdirAll(storachaAggregateStagedDir(targetPath), 0755); err != nil { + return "", xerrors.Errorf("creating aggregate staged marker directory: %w", err) + } + + stagingPath := storachaAggregateStagingPath(targetPath, verified.PieceCIDV2) + if filepath.Clean(tempPath) == filepath.Clean(stagingPath) { + return "", xerrors.Errorf("aggregate group %d is already at staging path %s", group.GroupID, stagingPath) + } + + if info, err := os.Stat(stagingPath); err == nil { + if info.IsDir() { + return "", xerrors.Errorf("aggregate staging path is a directory: %s", stagingPath) + } + if info.Size() < 0 || uint64(info.Size()) != verified.RawSize { + return "", xerrors.Errorf("aggregate staging file %s size mismatch: expected %d, got %d; cleanup manually before retrying", stagingPath, verified.RawSize, info.Size()) + } + marked, err := storachaAggregateStagedMarkerExists(targetPath, verified.PieceCIDV2) + if err != nil { + return "", err + } + if marked { + return "", xerrors.Errorf("committed aggregate staging file already exists: %s; retry should recover it before staging another aggregate", stagingPath) + } + return "", xerrors.Errorf("uncommitted aggregate staging file already exists: %s; cleanup manually before retrying", stagingPath) + } else if !os.IsNotExist(err) { + return "", xerrors.Errorf("checking aggregate staging path %s: %w", stagingPath, err) + } + + if err := os.Rename(tempPath, stagingPath); err != nil { + return "", xerrors.Errorf("moving aggregate group %d to staging: %w", group.GroupID, err) + } + if err := syncDir(filepath.Dir(tempPath)); err != nil { + return "", err + } + if err := syncDir(staging); err != nil { + return "", err + } + if err := writeStorachaAggregateStagedMarker(targetPath, verified.PieceCIDV2); err != nil { + return "", err + } + + return stagingPath, nil +} + +func importStagedStorachaAggregateFile(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, workDir string, group storachaAggregateGroup, stagingPath string, verified storachaVerifiedAggregatePiece) error { + // This function assumes staging has already been committed by + // staged/. It does not create or refresh that marker, and it does + // not check it again or recompute CommP. It only performs cheap consistency + // checks, imports the staged file through the existing import-pieces path, then + // advances the group to complete/.json. + expectedStagingPath := storachaAggregateStagingPath(targetPath, verified.PieceCIDV2) + if filepath.Clean(stagingPath) != filepath.Clean(expectedStagingPath) { + return xerrors.Errorf("aggregate staging path mismatch: got %s, expected %s", stagingPath, expectedStagingPath) + } + + info, err := os.Stat(stagingPath) + if err != nil { + return xerrors.Errorf("checking aggregate staging file %s: %w", stagingPath, err) + } + if info.IsDir() { + return xerrors.Errorf("aggregate staging path is a directory: %s", stagingPath) + } + if info.Size() < 0 || uint64(info.Size()) != verified.RawSize { + return xerrors.Errorf("aggregate staging file %s size mismatch: expected %d, got %d; cleanup manually before retrying", stagingPath, verified.RawSize, info.Size()) + } + + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath, false) + if err != nil { + return err + } + if !result.Imported { + return xerrors.Errorf("aggregate piece %s is not importable yet", verified.PieceCIDV2) + } + + if err := finishStorachaAggregateGroup(targetPath, workDir, group, verified); err != nil { + return err + } + return nil +} + +func openStorachaAggregateSubPieceReaders(sourcePath string, subPieces []storachaBucketRecord) ([]io.Reader, func() error, error) { + // datasegment consumes one reader per subpiece in group order. We cap each + // reader at the raw size recorded from the PieceCIDv2 so stray bytes in a + // source file cannot affect the aggregate stream. + readers := make([]io.Reader, 0, len(subPieces)) + files := make([]*os.File, 0, len(subPieces)) + + closeReaders := func() error { + var err error + for _, file := range files { + err = errors.Join(err, file.Close()) + } + return err + } + + for _, piece := range subPieces { + // The source is read-only and is addressed exactly like Curio piece park: + // source/piece/s-t00-. We do not fall back to CID-named + // files here because aggregate-pieces is replaying already imported + // parked pieces, not consuming import-pieces source files. + piecePath := storachaFinalPiecePath(sourcePath, piece.PieceID) + file, err := os.Open(piecePath) + if err != nil { + _ = closeReaders() + return nil, nil, xerrors.Errorf("opening subpiece %s: %w", piecePath, err) + } + files = append(files, file) + + info, err := file.Stat() + if err != nil { + _ = closeReaders() + return nil, nil, xerrors.Errorf("checking subpiece %s: %w", piecePath, err) + } + if info.IsDir() { + _ = closeReaders() + return nil, nil, xerrors.Errorf("subpiece path is a directory: %s", piecePath) + } + if info.Size() < 0 || uint64(info.Size()) != piece.RawSize { + _ = closeReaders() + return nil, nil, xerrors.Errorf("subpiece %s has size %d, expected %d", piecePath, info.Size(), piece.RawSize) + } + + readers = append(readers, io.LimitReader(file, int64(piece.RawSize))) + } + + return readers, closeReaders, nil +} + +func writeStorachaAggregateCompletion(workDir string, group storachaAggregateGroup, verified storachaVerifiedAggregatePiece) error { + // This marker is written last. After it exists, retries skip the group and + // result generation reports the aggregate to the caller. + subPieces := make([]string, 0, len(group.Pieces)) + for _, piece := range group.Pieces { + subPieces = append(subPieces, piece.PieceCIDV2) + } + + complete := storachaAggregateCompletion{ + GroupID: group.GroupID, + PieceCID: verified.PieceCIDV2.String(), + SubPieces: subPieces, + Count: len(subPieces), + PieceCIDV1: group.PieceCIDV1, + RawSize: verified.RawSize, + PaddedSize: group.PaddedSize, + } + if err := writeJSONFileAtomic(storachaAggregateCompletionPath(workDir, group.GroupID), complete); err != nil { + return xerrors.Errorf("writing aggregate completion marker for group %d: %w", group.GroupID, err) + } + return nil +} + +func finishStorachaAggregateGroup(targetPath, workDir string, group storachaAggregateGroup, verified storachaVerifiedAggregatePiece) error { + if err := removeStorachaAggregateStagedMarker(targetPath, verified.PieceCIDV2); err != nil { + return err + } + return writeStorachaAggregateCompletion(workDir, group, verified) +} + +func writeStorachaAggregateStagedMarker(targetPath string, pcidV2 cid.Cid) error { + markerPath := storachaAggregateStagedMarkerPath(targetPath, pcidV2) + if err := os.MkdirAll(filepath.Dir(markerPath), 0755); err != nil { + return xerrors.Errorf("creating aggregate staged marker directory: %w", err) + } + + tmpPath := markerPath + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating aggregate staged marker temp %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(syncErr, closeErr); err != nil { + return xerrors.Errorf("writing aggregate staged marker temp %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, markerPath); err != nil { + return xerrors.Errorf("renaming aggregate staged marker %s: %w", markerPath, err) + } + removeTmp = false + if err := syncDir(filepath.Dir(markerPath)); err != nil { + return err + } + return nil +} + +func storachaAggregateStagedMarkerExists(targetPath string, pcidV2 cid.Cid) (bool, error) { + markerPath := storachaAggregateStagedMarkerPath(targetPath, pcidV2) + info, err := os.Stat(markerPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, xerrors.Errorf("checking aggregate staged marker %s: %w", markerPath, err) + } + if info.IsDir() { + return false, xerrors.Errorf("aggregate staged marker is a directory: %s; cleanup manually before retrying", markerPath) + } + return true, nil +} + +func removeStorachaAggregateStagedMarker(targetPath string, pcidV2 cid.Cid) error { + markerPath := storachaAggregateStagedMarkerPath(targetPath, pcidV2) + if err := os.Remove(markerPath); err != nil { + if os.IsNotExist(err) { + return nil + } + return xerrors.Errorf("removing aggregate staged marker %s: %w", markerPath, err) + } + if err := syncDir(filepath.Dir(markerPath)); err != nil { + return err + } + return nil +} + +func storachaAggregateStagingDir(targetPath string) string { + return filepath.Join(targetPath, storachaAggregateStagingDirName) +} + +func storachaAggregateStagedDir(targetPath string) string { + return filepath.Join(storachaAggregateStagingDir(targetPath), storachaAggregateStagedDirName) +} + +func storachaAggregateStagingPath(targetPath string, pcidV2 cid.Cid) string { + return filepath.Join(storachaAggregateStagingDir(targetPath), pcidV2.String()+".car") +} + +func storachaAggregateStagedMarkerPath(targetPath string, pcidV2 cid.Cid) string { + return filepath.Join(storachaAggregateStagedDir(targetPath), pcidV2.String()) +} + +func storachaAggregateCompletionPath(workDir string, groupID int64) string { + return filepath.Join(workDir, storachaAggregateCompleteDir, fmt.Sprintf("%0*d.json", storachaAggregateGroupIDWidth, groupID)) +} + +func storachaAggregateTempFilePath(workDir string, groupID int64) string { + return filepath.Join(workDir, storachaAggregateTempDir, fmt.Sprintf("%0*d.car.tmp", storachaAggregateGroupIDWidth, groupID)) +} + +func storachaAggregateBucketFileName(size abi.PaddedPieceSize) string { + return fmt.Sprintf("%d.jsonl", uint64(size)) +} + +func storachaAggregateBucketSizesAscending() []abi.PaddedPieceSize { + // Filecoin piece sizes are powers of two. For a 1-GiB aggregate target this + // creates bucket names from 128.jsonl through 1073741824.jsonl. + sizes := make([]abi.PaddedPieceSize, 0, 24) + for size := abi.PaddedPieceSize(128); size <= storachaAggregatePieceSize; size <<= 1 { + sizes = append(sizes, size) + } + return sizes +} + +func storachaAggregateBucketSizesDescending() []abi.PaddedPieceSize { + // Grouping consumes largest buckets first, but raw finalization/checksums are + // easier to reason about in ascending filename order. + ascending := storachaAggregateBucketSizesAscending() + descending := make([]abi.PaddedPieceSize, 0, len(ascending)) + for i := len(ascending) - 1; i >= 0; i-- { + descending = append(descending, ascending[i]) + } + return descending +} + +func storachaAggregateIsBucketSize(size abi.PaddedPieceSize) bool { + if size < 128 || size > storachaAggregatePieceSize { + return false + } + if err := size.Validate(); err != nil { + return false + } + return true +} + +func writeJSONLine(writer *bufio.Writer, value any) (int, error) { + line, err := json.Marshal(value) + if err != nil { + return 0, err + } + line = append(line, '\n') + return writer.Write(line) +} + +func writeAggregatePiecesResultFromWork(path, workDir string, runErr error) error { + // The result is reconstructed from completion markers, not from in-memory + // state. That makes it safe to report completed aggregates after retries, + // crashes, or a final failure in a later group. + if workDir == "" { + out := aggregatePiecesOutput{Aggregates: []aggregates{}} + if runErr != nil { + out.Error = runErr.Error() + } + return writeAggregatePiecesResult(path, out) + } + + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return xerrors.Errorf("creating aggregate result directory: %w", err) + } + + tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating aggregate result temp file: %w", err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + writer := bufio.NewWriterSize(file, 1024*1024) + if _, err := writer.WriteString("{\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if runErr != nil { + errJSON, err := json.Marshal(runErr.Error()) + if err != nil { + _ = file.Close() + return xerrors.Errorf("marshalling aggregate result error: %w", err) + } + if _, err := writer.WriteString(" \"error\": "); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if _, err := writer.Write(errJSON); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if _, err := writer.WriteString(",\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if _, err := writer.WriteString(" \"aggregates\": ["); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + + first := true + groupsPath := filepath.Join(workDir, storachaAggregateGroupsFile) + groupsFile, err := os.Open(groupsPath) + if err == nil { + // Stream groups.jsonl instead of loading it. There may be many aggregate + // groups, but only completed groups produce result entries. + reader := bufio.NewReaderSize(groupsFile, 1024*1024) + for { + line, ok, readErr := readJSONLLine(reader) + if readErr != nil { + if runErr != nil { + break + } + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("reading aggregate groups for result: %w", readErr) + } + if !ok { + break + } + + var group storachaAggregateGroup + if err := json.Unmarshal(line, &group); err != nil { + if runErr != nil { + break + } + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("decoding aggregate group for result: %w", err) + } + + complete, ok, err := readCompletedStorachaAggregateGroup(workDir, group) + if err != nil { + if runErr != nil { + continue + } + _ = groupsFile.Close() + _ = file.Close() + return err + } + if !ok { + continue + } + + result := struct { + PieceCID string `json:"piece_cid"` + SubPieces []string `json:"sub_pieces"` + Count int `json:"count"` + }{ + PieceCID: complete.PieceCID, + SubPieces: complete.SubPieces, + Count: complete.Count, + } + resultJSON, err := json.Marshal(result) + if err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("marshalling aggregate result: %w", err) + } + + if first { + if _, err := writer.WriteString("\n "); err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + first = false + } else { + if _, err := writer.WriteString(",\n "); err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if _, err := writer.Write(resultJSON); err != nil { + _ = groupsFile.Close() + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if err := groupsFile.Close(); err != nil { + _ = file.Close() + return xerrors.Errorf("closing aggregate groups for result: %w", err) + } + } else if !os.IsNotExist(err) { + _ = file.Close() + return xerrors.Errorf("opening aggregate groups for result: %w", err) + } + + if first { + if _, err := writer.WriteString("]\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } else { + if _, err := writer.WriteString("\n ]\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + } + if _, err := writer.WriteString("}\n"); err != nil { + _ = file.Close() + return xerrors.Errorf("writing aggregate result: %w", err) + } + if err := writer.Flush(); err != nil { + _ = file.Close() + return xerrors.Errorf("flushing aggregate result: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return xerrors.Errorf("syncing aggregate result: %w", err) + } + if err := file.Close(); err != nil { + return xerrors.Errorf("closing aggregate result: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return xerrors.Errorf("renaming aggregate result: %w", err) + } + removeTmp = false + return syncDir(filepath.Dir(path)) +} + +func writeAggregatePiecesResult(path string, out aggregatePiecesOutput) error { + if out.Aggregates == nil { + out.Aggregates = []aggregates{} + } + if err := writeJSONFileAtomic(path, out); err != nil { + return xerrors.Errorf("writing aggregate result: %w", err) + } + return nil +} + +func writeJSONFileAtomic(path string, value any) error { + // All small state files use the same tmp-file protocol: write, fsync, close, + // rename, then fsync the directory so the rename itself survives a crash. + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return xerrors.Errorf("creating directory for %s: %w", path, err) + } + + tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("creating temp JSON file %s: %w", tmpPath, err) + } + removeTmp := true + defer func() { + if removeTmp { + _ = os.Remove(tmpPath) + } + }() + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(value) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(encodeErr, syncErr, closeErr); err != nil { + return xerrors.Errorf("writing temp JSON file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, path); err != nil { + return xerrors.Errorf("renaming temp JSON file %s to %s: %w", tmpPath, path, err) + } + removeTmp = false + return syncDir(filepath.Dir(path)) +} + +func syncDir(path string) error { + // Directory fsync is what makes a preceding rename durable on filesystems + // that otherwise may persist file contents without persisting the directory + // entry update. + dir, err := os.Open(path) + if err != nil { + return xerrors.Errorf("opening directory %s for sync: %w", path, err) + } + defer func() { + _ = dir.Close() + }() + if err := dir.Sync(); err != nil { + return xerrors.Errorf("syncing directory %s: %w", path, err) + } + return nil +} diff --git a/cmd/curio/storacha-migration-aggregate_test.go b/cmd/curio/storacha-migration-aggregate_test.go new file mode 100644 index 000000000..4d5ef03d1 --- /dev/null +++ b/cmd/curio/storacha-migration-aggregate_test.go @@ -0,0 +1,496 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/market/mk20" +) + +type storachaAggregateTestPiece struct { + storachaPieceFixture + parkedID int64 +} + +func TestStorachaAggregateDedupeAndCompletedMarkerResume(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2, pieces[0].cidV2) + + workDir, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + manifest := readStorachaAggregateManifestForTest(t, workDir) + require.Equal(t, int64(3), manifest.RawBuckets.Records) + require.Equal(t, int64(2), manifest.DedupedBuckets.Records) + require.Equal(t, int64(2), manifest.Groups.Pieces) + + group := readStorachaAggregateGroupForTest(t, workDir, 0) + require.Len(t, group.Pieces, 2) + complete := assertStorachaAggregateImported(t, env, workDir, group) + + // The completion marker is the resume boundary after a group is fully + // imported. Remove the source subpieces to prove a rerun trusts the marker + // instead of trying to read source bytes again. + for _, piece := range pieces { + require.NoError(t, os.Remove(storachaFinalPiecePath(env.sourceDir, piece.parkedID))) + } + + _, err = runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + require.Equal(t, complete, readStorachaAggregateCompletionForTest(t, workDir, group)) +} + +func TestStorachaAggregateRawBucketCursorTruncatesTornTail(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + inputSHA256, _, err := hashFileSHA256(inputPath) + require.NoError(t, err) + + workDir := filepath.Join(env.targetDir, storachaAggregateWorkDir, inputSHA256) + rawDir := filepath.Join(workDir, storachaAggregateRawBucketDir) + require.NoError(t, os.MkdirAll(rawDir, 0755)) + + firstRecord := storachaAggregateBucketRecordForTest(pieces[0]) + firstRecordLine, err := json.Marshal(firstRecord) + require.NoError(t, err) + firstRecordLine = append(firstRecordLine, '\n') + + rawBucketName := storachaAggregateBucketFileName(abi.PaddedPieceSize(pieces[0].info.Size)) + rawBucketPath := filepath.Join(rawDir, rawBucketName) + writeFile(t, rawBucketPath, append(firstRecordLine, []byte(`{"piece_cid_v2":"torn"`)...)) + require.NoError(t, writeJSONFileAtomic(filepath.Join(workDir, storachaAggregateBucketCursor), storachaAggregateBucketCursorState{ + InputSHA256: inputSHA256, + InputOffset: int64(len(pieces[0].cidV2) + 1), + InputLine: 1, + Records: 1, + BucketOffsets: map[string]int64{ + rawBucketName: int64(len(firstRecordLine)), + }, + })) + + actualWorkDir, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + require.Equal(t, workDir, actualWorkDir) + + rawBytes, err := os.ReadFile(rawBucketPath) + require.NoError(t, err) + require.NotContains(t, string(rawBytes), "torn") + + manifest := readStorachaAggregateManifestForTest(t, workDir) + require.Equal(t, int64(2), manifest.RawBuckets.Records) + require.Equal(t, int64(2), manifest.DedupedBuckets.Records) + assertStorachaAggregateImported(t, env, workDir, readStorachaAggregateGroupForTest(t, workDir, 0)) +} + +func TestStorachaAggregateIgnoresExistingTempFile(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath := storachaAggregateTempFilePath(workDir, group.GroupID) + writeFile(t, tempPath, []byte("leftover temp data")) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + assertMissing(t, tempPath) + assertStorachaAggregateImported(t, env, workDir, group) +} + +func TestStorachaAggregateRecoversFromStagingFile(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + stagingDir := storachaAggregateStagingDir(env.targetDir) + require.NoError(t, os.MkdirAll(stagingDir, 0755)) + stagingPath := storachaAggregateStagingPath(env.targetDir, verified.PieceCIDV2) + require.NoError(t, os.Rename(tempPath, stagingPath)) + require.NoError(t, syncDir(stagingDir)) + require.NoError(t, writeStorachaAggregateStagedMarker(env.targetDir, verified.PieceCIDV2)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + assertMissing(t, stagingPath) + assertMissing(t, storachaAggregateStagedMarkerPath(env.targetDir, verified.PieceCIDV2)) + complete := assertStorachaAggregateImported(t, env, workDir, group) + require.Equal(t, verified.PieceCIDV2.String(), complete.PieceCID) +} + +func TestStorachaAggregateRecoversFromFinalFileAndFixesDB(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + pieceID := insertAggregateParkedPiece(t, env, verified, false) + assertStorachaRefs(t, env, pieceID, 0) + assertStorachaRefsWithDataURL(t, env, pieceID, storachaMigrationAggregateDataURL, 0) + assertSectorLocation(t, env, pieceID, 0) + + finalPath := storachaFinalPiecePath(env.targetDir, pieceID) + require.NoError(t, os.Rename(tempPath, finalPath)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.NoError(t, err) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + assertStorachaRefs(t, env, pieceID, 0) + assertStorachaRefsWithDataURL(t, env, pieceID, storachaMigrationAggregateDataURL, 1) + assertPDPRefs(t, env, pieceID, group.PieceCIDV1, 0) + assertSectorLocation(t, env, pieceID, 1) + + complete := assertStorachaAggregateImported(t, env, workDir, group) + require.Equal(t, verified.PieceCIDV2.String(), complete.PieceCID) +} + +func TestStorachaAggregateUncommittedStagingRequiresManualCleanup(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + require.NoError(t, os.Remove(tempPath)) + + stagingDir := storachaAggregateStagingDir(env.targetDir) + require.NoError(t, os.MkdirAll(stagingDir, 0755)) + stagingPath := storachaAggregateStagingPath(env.targetDir, verified.PieceCIDV2) + writeFile(t, stagingPath, bytes.Repeat([]byte{0x7f}, int(verified.RawSize))) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.ErrorContains(t, err, "uncommitted aggregate staging file already exists") + require.ErrorContains(t, err, "cleanup manually") + assertMissing(t, storachaAggregateCompletionPath(workDir, group.GroupID)) +} + +func TestStorachaAggregateStagedMarkerMissingFileRequiresManualCleanup(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + require.NoError(t, os.Remove(tempPath)) + require.NoError(t, writeStorachaAggregateStagedMarker(env.targetDir, verified.PieceCIDV2)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.ErrorContains(t, err, "staged marker") + require.ErrorContains(t, err, "is missing") + require.ErrorContains(t, err, "cleanup manually") + assertMissing(t, storachaAggregateCompletionPath(workDir, group.GroupID)) +} + +func TestStorachaAggregateFinalSizeMismatchRequiresManualCleanup(t *testing.T) { + withSmallStorachaAggregateSize(t) + + env := setupStorachaMigrationTest(t) + pieces := prepareStorachaAggregateSourcePieces(t, env, 2) + inputPath := writeStorachaAggregateInput(t, pieces[0].cidV2, pieces[1].cidV2) + workDir, group := prepareStorachaAggregateGroupsForTest(t, env, inputPath) + + tempPath, verified := createStorachaAggregateTempFileForTest(t, env, workDir, group) + require.NoError(t, os.Remove(tempPath)) + + pieceID := insertAggregateParkedPiece(t, env, verified, false) + writeFile(t, storachaFinalPiecePath(env.targetDir, pieceID), bytes.Repeat([]byte{0x42}, int(verified.RawSize)-1)) + + _, err := runAggregatePieces(env.ctx, env.db, env.si, env.storageID, storachaAggregateSortPath(t), inputPath, env.sourceDir, env.targetDir) + require.ErrorContains(t, err, "size mismatch") + require.ErrorContains(t, err, "cleanup manually") + assertMissing(t, storachaAggregateCompletionPath(workDir, group.GroupID)) + require.False(t, parkedPieceByID(t, env, pieceID).complete) +} + +func withSmallStorachaAggregateSize(t *testing.T) { + t.Helper() + + previous := storachaAggregatePieceSize + storachaAggregatePieceSize = abi.PaddedPieceSize(4 << 20) + t.Cleanup(func() { + storachaAggregatePieceSize = previous + }) +} + +func prepareStorachaAggregateSourcePieces(t *testing.T, env storachaMigrationTestEnv, count int) []storachaAggregateTestPiece { + t.Helper() + + pieces := make([]storachaAggregateTestPiece, 0, count) + for i := 0; i < count; i++ { + fx := newValidStorachaAggregatePieceFixture(t, byte(i+1)) + pieceID := insertParkedPiece(t, env, fx, true, true, nil) + writeFile(t, storachaFinalPiecePath(env.sourceDir, pieceID), fx.data) + pieces = append(pieces, storachaAggregateTestPiece{ + storachaPieceFixture: fx, + parkedID: pieceID, + }) + } + return pieces +} + +func newValidStorachaAggregatePieceFixture(t *testing.T, seed byte) storachaPieceFixture { + t.Helper() + + data := bytes.Repeat([]byte{seed}, 127) + cp := new(commp.Calc) + defer cp.Reset() + n, err := io.Copy(cp, bytes.NewReader(data)) + require.NoError(t, err) + + digest, paddedSize, err := cp.Digest() + require.NoError(t, err) + + pcidV2, err := commcid.DataCommitmentToPieceCidv2(digest, uint64(n)) + require.NoError(t, err) + + info, err := mk20.GetPieceInfo(pcidV2) + require.NoError(t, err) + require.Equal(t, uint64(n), info.RawSize) + require.Equal(t, abi.PaddedPieceSize(paddedSize), info.Size) + + return storachaPieceFixture{ + cidV2: pcidV2.String(), + fileName: pcidV2.String() + ".car", + info: info, + data: data, + } +} + +func writeStorachaAggregateInput(t *testing.T, pieceCIDs ...string) string { + t.Helper() + + inputPath := filepath.Join(t.TempDir(), "aggregate-input.txt") + writeFile(t, inputPath, []byte(strings.Join(pieceCIDs, "\n")+"\n")) + return inputPath +} + +func storachaAggregateSortPath(t *testing.T) string { + t.Helper() + + sortPath, err := exec.LookPath("sort") + require.NoError(t, err) + return sortPath +} + +func prepareStorachaAggregateGroupsForTest(t *testing.T, env storachaMigrationTestEnv, inputPath string) (string, storachaAggregateGroup) { + t.Helper() + + inputSHA256, inputSize, err := hashFileSHA256(inputPath) + require.NoError(t, err) + + workDir := filepath.Join(env.targetDir, storachaAggregateWorkDir, inputSHA256) + require.NoError(t, os.MkdirAll(workDir, 0755)) + + manifest, err := loadStorachaAggregateManifest(workDir, inputSHA256, inputSize) + require.NoError(t, err) + require.NoError(t, buildRawStorachaAggregateBuckets(env.ctx, env.db, inputPath, inputSHA256, workDir, manifest)) + require.NoError(t, dedupeStorachaAggregateBuckets(env.ctx, storachaAggregateSortPath(t), workDir, manifest)) + require.NoError(t, buildStorachaAggregateGroups(workDir, manifest)) + + return workDir, readStorachaAggregateGroupForTest(t, workDir, 0) +} + +func createStorachaAggregateTempFileForTest(t *testing.T, env storachaMigrationTestEnv, workDir string, group storachaAggregateGroup) (string, storachaVerifiedAggregatePiece) { + t.Helper() + + expectedCIDV1, err := cid.Parse(group.PieceCIDV1) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(workDir, storachaAggregateTempDir), 0755)) + + tempPath, verified, err := createStorachaAggregateTempFile(env.sourceDir, workDir, group, expectedCIDV1, abi.PaddedPieceSize(group.PaddedSize)) + require.NoError(t, err) + require.Equal(t, group.PieceCIDV1, verified.PieceCIDV1.String()) + require.Equal(t, abi.PaddedPieceSize(group.PaddedSize), verified.PaddedSize) + return tempPath, verified +} + +func readStorachaAggregateManifestForTest(t *testing.T, workDir string) storachaAggregateWorkManifest { + t.Helper() + + mb, err := os.ReadFile(filepath.Join(workDir, storachaAggregateManifestFile)) + require.NoError(t, err) + + var manifest storachaAggregateWorkManifest + require.NoError(t, json.Unmarshal(mb, &manifest)) + return manifest +} + +func readStorachaAggregateGroupForTest(t *testing.T, workDir string, groupID int64) storachaAggregateGroup { + t.Helper() + + file, err := os.Open(filepath.Join(workDir, storachaAggregateGroupsFile)) + require.NoError(t, err) + defer func() { + _ = file.Close() + }() + + reader := bufio.NewReader(file) + for { + line, ok, err := readJSONLLine(reader) + require.NoError(t, err) + require.True(t, ok, "group %d not found", groupID) + + var group storachaAggregateGroup + require.NoError(t, json.Unmarshal(line, &group)) + if group.GroupID == groupID { + return group + } + } +} + +func readStorachaAggregateCompletionForTest(t *testing.T, workDir string, group storachaAggregateGroup) storachaAggregateCompletion { + t.Helper() + + complete, ok, err := readCompletedStorachaAggregateGroup(workDir, group) + require.NoError(t, err) + require.True(t, ok) + return complete +} + +func assertStorachaAggregateImported(t *testing.T, env storachaMigrationTestEnv, workDir string, group storachaAggregateGroup) storachaAggregateCompletion { + t.Helper() + + complete := readStorachaAggregateCompletionForTest(t, workDir, group) + pcidV2, err := cid.Parse(complete.PieceCID) + require.NoError(t, err) + pcidV1, rawSize, err := commcid.PieceCidV1FromV2(pcidV2) + require.NoError(t, err) + require.Equal(t, group.PieceCIDV1, pcidV1.String()) + require.Equal(t, complete.RawSize, rawSize) + + var rows []storachaFinalRecoveryRow + err = env.db.Select(env.ctx, &rows, ` + SELECT id, piece_cid, piece_padded_size, piece_raw_size + FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND piece_raw_size = $3 + AND long_term = TRUE + AND cleanup_task_id IS NULL + ORDER BY id`, group.PieceCIDV1, int64(group.PaddedSize), int64(complete.RawSize)) + require.NoError(t, err) + require.Len(t, rows, 1) + + pp := parkedPieceByID(t, env, rows[0].ID) + require.True(t, pp.complete) + assertStorachaRefs(t, env, rows[0].ID, 0) + assertStorachaRefsWithDataURL(t, env, rows[0].ID, storachaMigrationAggregateDataURL, 1) + assertPDPRefs(t, env, rows[0].ID, group.PieceCIDV1, 0) + assertSectorLocation(t, env, rows[0].ID, 1) + + _, err = verifyStorachaAggregateFileForTest(storachaFinalPiecePath(env.targetDir, rows[0].ID), pcidV1, abi.PaddedPieceSize(group.PaddedSize)) + require.NoError(t, err) + return complete +} + +func verifyStorachaAggregateFileForTest(path string, expectedCIDV1 cid.Cid, expectedPaddedSize abi.PaddedPieceSize) (storachaVerifiedAggregatePiece, error) { + file, err := os.Open(path) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + defer func() { + _ = file.Close() + }() + + info, err := file.Stat() + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + if info.IsDir() { + return storachaVerifiedAggregatePiece{}, os.ErrInvalid + } + + cp := new(commp.Calc) + defer cp.Reset() + n, err := io.Copy(cp, file) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + + digest, actualPaddedSize, err := cp.Digest() + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + if abi.PaddedPieceSize(actualPaddedSize) != expectedPaddedSize { + return storachaVerifiedAggregatePiece{}, os.ErrInvalid + } + + actualCIDV1, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + if !actualCIDV1.Equals(expectedCIDV1) { + return storachaVerifiedAggregatePiece{}, os.ErrInvalid + } + + actualCIDV2, err := commcid.DataCommitmentToPieceCidv2(digest, uint64(n)) + if err != nil { + return storachaVerifiedAggregatePiece{}, err + } + + return storachaVerifiedAggregatePiece{ + PieceCIDV2: actualCIDV2, + PieceCIDV1: actualCIDV1, + RawSize: uint64(n), + PaddedSize: abi.PaddedPieceSize(actualPaddedSize), + }, nil +} + +func insertAggregateParkedPiece(t *testing.T, env storachaMigrationTestEnv, verified storachaVerifiedAggregatePiece, complete bool) int64 { + t.Helper() + + var pieceID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, complete, long_term, skip) + VALUES ($1, $2, $3, $4, TRUE, TRUE) + RETURNING id + `, verified.PieceCIDV1.String(), int64(verified.PaddedSize), int64(verified.RawSize), complete).Scan(&pieceID) + require.NoError(t, err) + return pieceID +} + +func storachaAggregateBucketRecordForTest(piece storachaAggregateTestPiece) storachaBucketRecord { + return storachaBucketRecord{ + PieceCIDV2: piece.cidV2, + PieceID: piece.parkedID, + PieceCIDV1: piece.info.PieceCIDV1.String(), + RawSize: piece.info.RawSize, + PaddedSize: uint64(piece.info.Size), + } +} diff --git a/cmd/curio/storacha-migration.go b/cmd/curio/storacha-migration.go new file mode 100644 index 000000000..cb01498ac --- /dev/null +++ b/cmd/curio/storacha-migration.go @@ -0,0 +1,738 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ipfs/go-cid" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/alertmanager/curioalerting" + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/parkpiece" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" +) + +const minSaveCacheSize = abi.PaddedPieceSize(uint64(32 * 1024 * 1024)) +const serviceName = "public" +const storachaMigrationDataURL = "storacha-migration" + +type importPiecesOutput struct { + Count int `json:"count"` + Pieces []string `json:"pieces"` + Error string `json:"error,omitempty"` +} + +type storachaImportResult struct { + Imported bool + PieceCIDV2 string +} + +type storachaParkedPieceState struct { + ID int64 + UseStagedFile bool + AlreadyComplete bool +} + +type storachaFinalRecoveryRow struct { + ID int64 `db:"id"` + PieceCID string `db:"piece_cid"` + PaddedSize int64 `db:"piece_padded_size"` + RawSize int64 `db:"piece_raw_size"` +} + +var importPiecesCmd = &cli.Command{ + Name: "import-pieces", + Usage: "Imports already existing pieces from storage to piece park system", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "source", + Usage: "path to store the pieces from", + }, + &cli.StringFlag{ + Name: "target", + Usage: "path to storage directory in Curio's attached permanent storage", + }, + &cli.StringFlag{ + Name: "result", + Usage: "path to write the JSON result", + }, + &cli.IntFlag{ + Name: "batch-size", + Usage: "number of pieces to move to permanent storage", + Value: 20, + }, + }, + Action: func(cctx *cli.Context) (err error) { + ctx := cctx.Context + var out importPiecesOutput + + /* + How this migration works: + + The source directory has Storacha CAR files named by piece CID v2. Curio's + piece park stores the same bytes under a parked_pieces id, so the final + file under the target storage root becomes piece/s-t00-. + That means the migration must first learn or create the parked_pieces + row before it knows the final filename. + + Every fresh source file is moved to target/storacha-staging first. That + staging move is the first retry boundary: once the file leaves source, a + later run can still find it and finish the DB/ref/final-file work through + importStagedStorachaPiece. + + importStagedStorachaPiece owns the normal path: + 1. Parse the staged filename to get the piece info. + 2. In one DB transaction, create or reuse the parked_pieces row. + 3. If the row is incomplete and has a live harmony task, leave the staged + file alone because that task owns the normal download/write path. + 4. Otherwise, ensure exactly one storacha-migration parked_piece_refs row + and exactly one pdp_piecerefs row for that parked ref. + 5. After the transaction commits, move the staged file to + target/piece/s-t00-, declare it in sector_location, then mark + parked_pieces.complete=true. + + recoverFinalStorachaPieces exists for the one point where staging can no + longer help: if a previous run already renamed the file into target/piece + and then crashed before declaring the file or marking the parked piece + complete. At that point there is no staged filename left to process, so + recovery starts from the DB row/ref and checks whether the final file is + already present. + */ + + if cctx.String("result") == "" { + return fmt.Errorf("result is required") + } + + resultPath, err := homedir.Expand(cctx.String("result")) + if err != nil { + return xerrors.Errorf("expanding result path: %w", err) + } + resultPath, err = filepath.Abs(filepath.Clean(resultPath)) + if err != nil { + return xerrors.Errorf("resolving result path: %w", err) + } + + defer func() { + if err != nil { + out.Error = err.Error() + } + writeErr := writeImportPiecesResult(resultPath, out) + if writeErr != nil { + err = errors.Join(err, writeErr) + } + }() + + if cctx.String("source") == "" || cctx.String("target") == "" { + return fmt.Errorf("source and target both are required") + } + + sourcePath, err := homedir.Expand(cctx.String("source")) + if err != nil { + return xerrors.Errorf("expanding source path: %w", err) + } + + targetPath, err := homedir.Expand(cctx.String("target")) + if err != nil { + return xerrors.Errorf("expanding target path: %w", err) + } + targetPath, err = filepath.Abs(filepath.Clean(targetPath)) + if err != nil { + return xerrors.Errorf("resolving target path: %w", err) + } + + storageID, err := pieceStorageID(targetPath) + if err != nil { + return xerrors.Errorf("getting storage ID: %w", err) + } + + db, err := deps.MakeDB(cctx) + if err != nil { + return err + } + + si := paths.NewDBIndex(curioalerting.NewAlertingSystem(db), db) + + out, err = runImportPieces(ctx, db, si, storageID, sourcePath, targetPath, cctx.Int("batch-size")) + if err != nil { + return err + } + + return nil + }, +} + +func writeImportPiecesResult(path string, out importPiecesOutput) error { + resJson, err := json.MarshalIndent(out, "", " ") + if err != nil { + return xerrors.Errorf("marshalling import result: %w", err) + } + + if err := os.WriteFile(path, resJson, 0644); err != nil { + return xerrors.Errorf("writing import result: %w", err) + } + + return nil +} + +func runImportPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, sourcePath, targetPath string, batchSize int) (importPiecesOutput, error) { + var out importPiecesOutput + + staging := filepath.Join(targetPath, "storacha-staging") + existing, err := os.ReadDir(staging) + if err != nil { + if !os.IsNotExist(err) { + return out, xerrors.Errorf("reading staging directory: %w", err) + } + + err = os.MkdirAll(staging, 0755) + if err != nil { + return out, xerrors.Errorf("creating staging directory: %w", err) + } + } + + if len(existing) > 0 { + // Staging is the retry boundary: a file here was already moved out of + // source, but DB refs or final piece declaration may not be complete. + for _, entry := range existing { + if out.Count >= batchSize { + break + } + if entry.IsDir() { + continue + } + + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, filepath.Join(staging, entry.Name()), true) + if err != nil { + return out, err + } + if result.Imported { + out.Pieces = append(out.Pieces, result.PieceCIDV2) + out.Count++ + } + } + } + + // This covers the crash point after staging was renamed to the final + // piece path, but before sector_location/parked_pieces was finalized. + err = recoverFinalStorachaPieces(ctx, db, si, storageID, targetPath, &out, batchSize) + if err != nil { + return out, err + } + + // Open the directory and read all .car files in it. No need to care about subdirectories + entries, err := os.ReadDir(sourcePath) + if err != nil { + return out, xerrors.Errorf("reading directory: %w", err) + } + + for _, entry := range entries { + if out.Count >= batchSize { + break + } + if entry.IsDir() { + continue + } + + if !strings.HasSuffix(entry.Name(), ".car") { + continue + } + + _, ok, err := storachaPieceInfoFromFileName(entry.Name()) + if err != nil { + return out, err + } + if !ok { + continue + } + + oldPath := filepath.Join(sourcePath, entry.Name()) + stagingPath := filepath.Join(staging, entry.Name()) + if _, err := os.Stat(stagingPath); err == nil { + return out, xerrors.Errorf("staging file already exists: %s", stagingPath) + } else if !os.IsNotExist(err) { + return out, xerrors.Errorf("checking staging file: %w", err) + } + + // New imports also go through staging, so the same recovery code can + // resume after a crash at any later step. + err = os.Rename(oldPath, stagingPath) + if err != nil { + return out, xerrors.Errorf("renaming old path: %w", err) + } + + result, err := importStagedStorachaPiece(ctx, db, si, storageID, targetPath, stagingPath, true) + if err != nil { + return out, err + } + if result.Imported { + out.Pieces = append(out.Pieces, result.PieceCIDV2) + out.Count++ + } + } + + return out, nil +} + +// pieceStorageID maps the user-provided target directory back to Curio's +// storage id. The target flag is expected to point at the storage root that +// contains sectorstore.json; final piece files are stored under target/piece. +func pieceStorageID(targetPath string) (storiface.ID, error) { + mb, err := os.ReadFile(filepath.Join(targetPath, paths.MetaFile)) + if err != nil { + return "", xerrors.Errorf("reading storage metadata for %s: %w", targetPath, err) + } + + var meta storiface.LocalStorageMeta + if err := json.Unmarshal(mb, &meta); err != nil { + return "", xerrors.Errorf("unmarshalling storage metadata for %s: %w", targetPath, err) + } + + if meta.ID != ("") { + return meta.ID, nil + } + + return "", xerrors.Errorf("no storage ID found for %s", targetPath) +} + +// storachaPieceInfoFromFileName accepts only Storacha migration inputs named +// .car. Invalid names or unparsable CIDs are skipped; valid names +// are converted to the v1 piece CID plus padded/raw sizes used by parked_pieces. +func storachaPieceInfoFromFileName(name string) (*mk20.PieceInfo, bool, error) { + pcidStr, ok := strings.CutSuffix(name, ".car") + if !ok || pcidStr == "" { + return nil, false, nil + } + + pcid2, err := cid.Parse(pcidStr) + if err != nil { + return nil, false, nil + } + + pi, err := mk20.GetPieceInfo(pcid2) + if err != nil { + return nil, false, xerrors.Errorf("getting piece info: %w", err) + } + + return pi, true, nil +} + +// importStagedStorachaPiece finishes one target/storacha-staging/.car. +// It first creates or reuses parked_pieces/refs in a committed DB transaction. +// makePDPPieceRef keeps import-pieces behavior unchanged while letting aggregate +// imports use the aggregate migration data_url and skip the public PDP piece ref. +// If an existing incomplete parked_piece still has a live task, it returns +// Imported=false and leaves the staged file untouched for a later retry. +// Otherwise it consumes the staged file: remove it if the final +// target/piece/s-t00- already exists, or rename it to that final path. It +// then declares the piece location and marks parked_pieces.complete unless the +// row was already complete. +func importStagedStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath, stagingPath string, makePDPPieceRef bool) (storachaImportResult, error) { + info, err := os.Stat(stagingPath) + if err != nil { + return storachaImportResult{}, xerrors.Errorf("checking staging file: %w", err) + } + if info.IsDir() { + return storachaImportResult{}, nil + } + + pi, ok, err := storachaPieceInfoFromFileName(filepath.Base(stagingPath)) + if err != nil { + return storachaImportResult{}, err + } + if !ok { + return storachaImportResult{}, nil + } + + var state storachaParkedPieceState + // The transaction only decides ownership and DB references. It deliberately + // does not move the file to s-t00-; the id must be committed before the + // filename depends on it. + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + parkedPieceID, createdParkedPiece, err := parkpiece.UpsertSkipWithInserted(tx, pi.PieceCIDV1.String(), int64(pi.Size), int64(pi.RawSize), true, true) + if err != nil { + return false, xerrors.Errorf("upsert parked piece: %w", err) + } + + state = storachaParkedPieceState{ + ID: parkedPieceID, + UseStagedFile: createdParkedPiece, + } + + if !createdParkedPiece { + state, err = claimExistingStorachaPieceTx(tx, parkedPieceID) + if err != nil { + return false, err + } + } + + if !state.UseStagedFile { + return true, nil + } + + if err := ensureStorachaRefsTx(tx, parkedPieceID, pi, makePDPPieceRef); err != nil { + return false, err + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return storachaImportResult{}, xerrors.Errorf("upsert parked piece: %w", err) + } + if !comm { + return storachaImportResult{}, xerrors.Errorf("failed to commit transaction") + } + if !state.UseStagedFile { + // An existing incomplete parked_piece still has a live harmony task. Leave + // the staged file in place; a later retry can use it if that task disappears. + return storachaImportResult{}, nil + } + + finalPath := storachaFinalPiecePath(targetPath, state.ID) + finalInfo, err := os.Stat(finalPath) + switch { + case err == nil: + // A final file means an earlier run got past rename. The staged file is + // now only a duplicate input for the same parked piece. + if finalInfo.IsDir() { + return storachaImportResult{}, xerrors.Errorf("final piece path is a directory: %s", finalPath) + } + if err := os.Remove(stagingPath); err != nil { + return storachaImportResult{}, xerrors.Errorf("removing staged duplicate: %w", err) + } + case os.IsNotExist(err): + // Normal post-commit path: the DB id is durable, so it is now safe for the + // final filename to depend on parked_pieces.id. + if err := os.Rename(stagingPath, finalPath); err != nil { + return storachaImportResult{}, xerrors.Errorf("renaming staging file: %w", err) + } + default: + return storachaImportResult{}, xerrors.Errorf("checking final piece file: %w", err) + } + + if err := declareStorachaPiece(ctx, si, storageID, state.ID); err != nil { + return storachaImportResult{}, err + } + + if !state.AlreadyComplete { + // complete is last. If we crash before this, final-file recovery will see + // s-t00-, declare it again idempotently, and mark the row complete. + if err := markParkedPieceComplete(ctx, db, state.ID); err != nil { + return storachaImportResult{}, err + } + } + + return storachaImportResult{Imported: true, PieceCIDV2: strings.TrimSuffix(filepath.Base(stagingPath), ".car")}, nil +} + +// recoverFinalStorachaPieces scans incomplete storacha-migration rows where the +// DB/ref side already exists. For each row it checks whether the file has +// already reached its final target/piece/s-t00- path. If yes, it finishes +// declaration and complete=true. If no final file exists, it leaves the row +// alone; there is nothing safe to do without either the staged file or the final +// file. +func recoverFinalStorachaPieces(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath string, out *importPiecesOutput, batchSize int) error { + if out.Count >= batchSize { + return nil + } + + var rows []storachaFinalRecoveryRow + err := db.Select(ctx, &rows, ` + SELECT pp.id, pp.piece_cid, pp.piece_padded_size, pp.piece_raw_size + FROM parked_pieces pp + WHERE pp.complete = FALSE + AND pp.skip = TRUE + AND pp.cleanup_task_id IS NULL + AND EXISTS ( + SELECT 1 + FROM parked_piece_refs ppr + WHERE ppr.piece_id = pp.id + AND ppr.data_url = $1 + AND ppr.long_term = TRUE + ) + ORDER BY pp.id + LIMIT $2`, storachaMigrationDataURL, batchSize-out.Count) + if err != nil { + return xerrors.Errorf("query final storacha pieces: %w", err) + } + + for _, row := range rows { + if out.Count >= batchSize { + break + } + + imported, err := recoverFinalStorachaPiece(ctx, db, si, storageID, targetPath, row, true) + if err != nil { + return err + } + if imported { + pcidV2, err := storachaPieceCIDV2FromRow(row) + if err != nil { + return err + } + out.Pieces = append(out.Pieces, pcidV2) + out.Count++ + } + } + + return nil +} + +func storachaPieceCIDV2FromRow(row storachaFinalRecoveryRow) (string, error) { + pcidV1, err := cid.Parse(row.PieceCID) + if err != nil { + return "", xerrors.Errorf("parsing piece cid %s: %w", row.PieceCID, err) + } + + pcidV2, err := commcid.PieceCidV2FromV1(pcidV1, uint64(row.RawSize)) + if err != nil { + return "", xerrors.Errorf("getting piece cid v2: %w", err) + } + + return pcidV2.String(), nil +} + +// recoverFinalStorachaPiece handles the post-rename crash window for one DB row. +// It requires target/piece/s-t00- to exist, rebuilds the piece info from the +// DB row, reuses the same claim/ref checks as staged import, then declares the +// final file and marks the parked piece complete. It returns false when the +// final file is missing or a live task still owns the parked_piece. +func recoverFinalStorachaPiece(ctx context.Context, db *harmonydb.DB, si paths.SectorIndex, storageID storiface.ID, targetPath string, row storachaFinalRecoveryRow, makePDPPieceRef bool) (bool, error) { + finalPath := storachaFinalPiecePath(targetPath, row.ID) + info, err := os.Stat(finalPath) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, xerrors.Errorf("checking final piece file: %w", err) + } + if info.IsDir() { + return false, xerrors.Errorf("final piece path is a directory: %s", finalPath) + } + + pcid, err := cid.Parse(row.PieceCID) + if err != nil { + return false, xerrors.Errorf("parsing piece cid %s: %w", row.PieceCID, err) + } + pi := &mk20.PieceInfo{ + PieceCIDV1: pcid, + Size: abi.PaddedPieceSize(row.PaddedSize), + RawSize: uint64(row.RawSize), + } + + var state storachaParkedPieceState + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + state, err = claimExistingStorachaPieceTx(tx, row.ID) + if err != nil { + return false, err + } + if !state.UseStagedFile { + return true, nil + } + if err := ensureStorachaRefsTx(tx, row.ID, pi, makePDPPieceRef); err != nil { + return false, err + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("claim final storacha piece: %w", err) + } + if !comm { + return false, xerrors.Errorf("failed to commit transaction") + } + if !state.UseStagedFile { + return false, nil + } + + if err := declareStorachaPiece(ctx, si, storageID, row.ID); err != nil { + return false, err + } + if !state.AlreadyComplete { + if err := markParkedPieceComplete(ctx, db, row.ID); err != nil { + return false, err + } + } + + return true, nil +} + +// claimExistingStorachaPieceTx locks and classifies an existing parked_pieces +// row. Complete rows return AlreadyComplete=true and allow this migration to +// consume the staged file. Incomplete rows with a live harmony task return +// UseStagedFile=false. Incomplete rows without a live task are claimed by +// clearing task_id and setting skip=true, then this migration can finish them. +func claimExistingStorachaPieceTx(tx *harmonydb.Tx, parkedPieceID int64) (storachaParkedPieceState, error) { + var complete bool + var taskID sql.NullInt64 + var taskExists bool + err := tx.QueryRow(` + SELECT pp.complete, pp.task_id, ht.id IS NOT NULL AS task_exists + FROM parked_pieces pp + LEFT JOIN harmony_task ht ON ht.id = pp.task_id + WHERE pp.id = $1 + FOR UPDATE OF pp`, parkedPieceID).Scan(&complete, &taskID, &taskExists) + if err != nil { + return storachaParkedPieceState{}, xerrors.Errorf("query parked piece: %w", err) + } + + state := storachaParkedPieceState{ + ID: parkedPieceID, + AlreadyComplete: complete, + } + if complete { + state.UseStagedFile = true + return state, nil + } + if taskID.Valid && taskExists { + return state, nil + } + + n, err := tx.Exec(` + UPDATE parked_pieces + SET task_id = NULL, skip = TRUE + WHERE id = $1 + AND complete = FALSE + AND (task_id IS NULL OR task_id = $2)`, parkedPieceID, taskID.Int64) + if err != nil { + return storachaParkedPieceState{}, xerrors.Errorf("claim parked piece: %w", err) + } + if n != 1 { + return storachaParkedPieceState{}, xerrors.Errorf("claim parked piece: expected 1, got %d", n) + } + + state.UseStagedFile = true + return state, nil +} + +// ensureStorachaRefsTx enforces the intended one-to-one shape for this migration +// mode. Direct imports create storacha-migration refs and PDP refs; aggregate +// imports create storacha-migration-aggregate refs and intentionally skip PDP refs. +func ensureStorachaRefsTx(tx *harmonydb.Tx, parkedPieceID int64, pi *mk20.PieceInfo, makePDPPieceRef bool) error { + dataURL := storachaMigrationAggregateDataURL + if makePDPPieceRef { + dataURL = storachaMigrationDataURL + } + + var refs []struct { + RefID int64 `db:"ref_id"` + } + err := tx.Select(&refs, ` + SELECT ref_id + FROM parked_piece_refs + WHERE piece_id = $1 + AND data_url = $2 + AND long_term = TRUE + ORDER BY ref_id + FOR UPDATE`, parkedPieceID, dataURL) + if err != nil { + return xerrors.Errorf("query storacha parked piece refs: %w", err) + } + if len(refs) > 1 { + return xerrors.Errorf("storacha parked piece refs: expected at most 1 for parked piece %d, got %d", parkedPieceID, len(refs)) + } + + var pieceRef int64 + if len(refs) == 1 { + pieceRef = refs[0].RefID + } else { + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) + RETURNING ref_id`, parkedPieceID, dataURL).Scan(&pieceRef) + if err != nil { + return xerrors.Errorf("insert storacha parked piece ref: %w", err) + } + } + + if !makePDPPieceRef { + return nil + } + + return ensurePDPPieceRef(tx, pieceRef, pi) +} + +func ensurePDPPieceRef(tx *harmonydb.Tx, pieceRef int64, pi *mk20.PieceInfo) error { + var pdpRefs []struct { + ID int64 `db:"id"` + Service string `db:"service"` + PieceCID string `db:"piece_cid"` + } + err := tx.Select(&pdpRefs, ` + SELECT id, service, piece_cid + FROM pdp_piecerefs + WHERE piece_ref = $1 + FOR UPDATE`, pieceRef) + if err != nil { + return xerrors.Errorf("query pdp piece refs: %w", err) + } + if len(pdpRefs) > 1 { + return xerrors.Errorf("pdp piece refs: expected at most 1 for parked piece ref %d, got %d", pieceRef, len(pdpRefs)) + } + if len(pdpRefs) == 1 { + if pdpRefs[0].Service != serviceName || pdpRefs[0].PieceCID != pi.PieceCIDV1.String() { + return xerrors.Errorf("pdp piece ref %d does not match storacha migration piece %s", pdpRefs[0].ID, pi.PieceCIDV1.String()) + } + return nil + } + + needsSaveCache := padreader.PaddedSize(pi.RawSize).Padded() >= minSaveCacheSize + n, err := tx.Exec(` + INSERT INTO pdp_piecerefs (service, piece_cid, piece_ref, created_at, needs_save_cache) + VALUES ($1, $2, $3, NOW(), $4)`, serviceName, pi.PieceCIDV1.String(), pieceRef, needsSaveCache) + if err != nil { + return xerrors.Errorf("insert pdp piece ref: %w", err) + } + if n != 1 { + return xerrors.Errorf("insert pdp piece ref: expected 1, got %d", n) + } + + return nil +} + +// storachaFinalPiecePath mirrors Curio's piece-park sector naming: parked piece +// id N is stored as miner-zero piece sector s-t00-N under target/piece. +func storachaFinalPiecePath(targetPath string, parkedPieceID int64) string { + pieceNumber := storiface.PieceNumber(parkedPieceID) + return filepath.Join(targetPath, "piece", storiface.SectorName(pieceNumber.Ref().ID)) +} + +// declareStorachaPiece records the final piece file in sector_location. The +// storage index upsert is idempotent, so recovery can call this again after a +// crash between rename and complete=true. +func declareStorachaPiece(ctx context.Context, si paths.SectorIndex, storageID storiface.ID, parkedPieceID int64) error { + pieceNumber := storiface.PieceNumber(parkedPieceID) + err := si.StorageDeclareSector(ctx, storageID, pieceNumber.Ref().ID, storiface.FTPiece, true) + if err != nil { + return xerrors.Errorf("declaring storacha piece: %w", err) + } + return nil +} + +// markParkedPieceComplete is intentionally the last state transition. Once this +// is true, ParkPieceTask will not try to download/write the piece, so the final +// file and sector_location declaration must already be in place. +func markParkedPieceComplete(ctx context.Context, db *harmonydb.DB, parkedPieceID int64) error { + n, err := db.Exec(ctx, `UPDATE parked_pieces SET complete = TRUE WHERE id = $1`, parkedPieceID) + if err != nil { + return xerrors.Errorf("updating parked pieces: %w", err) + } + if n != 1 { + return xerrors.Errorf("updating parked pieces: expected 1, got %d", n) + } + return nil +} diff --git a/cmd/curio/storacha-migration_test.go b/cmd/curio/storacha-migration_test.go new file mode 100644 index 000000000..bdf5a1043 --- /dev/null +++ b/cmd/curio/storacha-migration_test.go @@ -0,0 +1,744 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + commcid "github.com/filecoin-project/go-fil-commcid" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" +) + +type storachaMigrationTestEnv struct { + ctx context.Context + db *harmonydb.DB + si paths.SectorIndex + storageID storiface.ID + sourceDir string + targetDir string + stagingDir string +} + +type storachaPieceFixture struct { + cidV2 string + fileName string + info *mk20.PieceInfo + data []byte +} + +type storachaPieceFixtures struct { + t *testing.T + seed uint64 +} + +type testParkedPiece struct { + id int64 + complete bool + skip bool + longTerm bool + taskID sql.NullInt64 +} + +type testPDPRef struct { + ID int64 `db:"id"` + Service string `db:"service"` + PieceCID string `db:"piece_cid"` + PieceRef int64 `db:"piece_ref"` +} + +func TestStorachaPieceInfoFromFileName(t *testing.T) { + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + + pi, ok, err := storachaPieceInfoFromFileName(fx.fileName) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, fx.info.PieceCIDV1.String(), pi.PieceCIDV1.String()) + require.Equal(t, fx.info.Size, pi.Size) + require.Equal(t, fx.info.RawSize, pi.RawSize) + + pi, ok, err = storachaPieceInfoFromFileName("not-a-car.txt") + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, pi) + + pi, ok, err = storachaPieceInfoFromFileName("not-a-cid.car") + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, pi) + + var commP [32]byte + pcidV1, err := commcid.DataCommitmentV1ToCID(commP[:]) + require.NoError(t, err) + pi, ok, err = storachaPieceInfoFromFileName(pcidV1.String() + ".car") + require.Error(t, err) + require.False(t, ok) + require.Nil(t, pi) +} + +func TestWriteImportPiecesResultIncludesError(t *testing.T) { + path := filepath.Join(t.TempDir(), "result.json") + expected := importPiecesOutput{ + Count: 2, + Pieces: []string{"piece-a", "piece-b"}, + Error: "import failed", + } + + require.NoError(t, writeImportPiecesResult(path, expected)) + + mb, err := os.ReadFile(path) + require.NoError(t, err) + + var actual importPiecesOutput + require.NoError(t, json.Unmarshal(mb, &actual)) + require.Equal(t, expected, actual) +} + +func TestStorachaMigrationFreshImport(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + writeSourcePiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{fx.cidV2}, out.Pieces) + require.NotEqual(t, []string{fx.info.PieceCIDV1.String()}, out.Pieces) + + pp := parkedPieceByCID(t, env, fx) + require.True(t, pp.complete) + require.True(t, pp.skip) + require.True(t, pp.longTerm) + require.False(t, pp.taskID.Valid) + + assertMissing(t, filepath.Join(env.sourceDir, fx.fileName)) + assertMissing(t, filepath.Join(env.stagingDir, fx.fileName)) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pp.id), fx.data) + assertStorachaRefs(t, env, pp.id, 1) + assertPDPRefs(t, env, pp.id, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pp.id, 1) +} + +func TestStorachaMigrationExistingCompleteMovesStagedWhenFinalMissing(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, true, false, nil) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + require.False(t, pp.skip) + assertMissing(t, filepath.Join(env.stagingDir, fx.fileName)) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationExistingCompleteRemovesStagedDuplicateWhenFinalExists(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, true, false, nil) + finalBytes := []byte("already-final") + writeStagedPiece(t, env, fx) + writeFinalPiece(t, env, pieceID, finalBytes) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + assertMissing(t, filepath.Join(env.stagingDir, fx.fileName)) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), finalBytes) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationClaimsIncompleteParkedPieceWithNoTask(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + require.True(t, pp.skip) + require.False(t, pp.taskID.Valid) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationClaimsIncompleteParkedPieceWithStaleTask(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + staleTaskID := int64(999999) + pieceID := insertParkedPiece(t, env, fx, false, false, &staleTaskID) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + require.True(t, pp.skip) + require.False(t, pp.taskID.Valid) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationLeavesLiveTaskPieceAlone(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + taskID := insertHarmonyTask(t, env, "live-park-piece") + pieceID := insertParkedPiece(t, env, fx, false, false, &taskID) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 0, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + require.False(t, pp.skip) + require.True(t, pp.taskID.Valid) + require.Equal(t, taskID, pp.taskID.Int64) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertStorachaRefs(t, env, pieceID, 0) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationReusesExistingRefs(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + pdpID := insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + refs := storachaRefIDs(t, env, pieceID) + require.Equal(t, []int64{refID}, refs) + pdpRefs := pdpRefsForPiece(t, env, pieceID) + require.Len(t, pdpRefs, 1) + require.Equal(t, pdpID, pdpRefs[0].ID) +} + +func TestStorachaMigrationInsertsMissingPDPRefForExistingParkedRef(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + writeStagedPiece(t, env, fx) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []int64{refID}, storachaRefIDs(t, env, pieceID)) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) +} + +func TestStorachaMigrationDuplicateStorachaRefsErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + insertStorachaParkedRef(t, env, pieceID) + insertStorachaParkedRef(t, env, pieceID) + writeStagedPiece(t, env, fx) + + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "expected at most 1") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationMismatchedPDPRefErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, "wrong-piece-cid") + writeStagedPiece(t, env, fx) + + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "does not match storacha migration piece") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationRecoversFinalFile(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + writeFinalPiece(t, env, pieceID, fx.data) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pieceID), fx.data) + assertStorachaRefs(t, env, pieceID, 1) + assertPDPRefs(t, env, pieceID, fx.info.PieceCIDV1.String(), 1) + assertSectorLocation(t, env, pieceID, 1) +} + +func TestStorachaMigrationFinalRecoveryMissingFileNoops(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 0, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationFinalRecoveryDirectoryErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) + + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "final piece path is a directory") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationFinalRecoveryLeavesLiveTaskPieceAlone(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + taskID := insertHarmonyTask(t, env, "live-final") + pieceID := insertParkedPiece(t, env, fx, false, true, &taskID) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, fx.info.PieceCIDV1.String()) + writeFinalPiece(t, env, pieceID, fx.data) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.NoError(t, err) + require.Equal(t, 0, out.Count) + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + require.True(t, pp.taskID.Valid) + require.Equal(t, taskID, pp.taskID.Int64) + assertSectorLocation(t, env, pieceID, 0) +} + +func TestStorachaMigrationBatchProcessesStagingBeforeSource(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + staged := pieces.next() + source := pieces.next() + writeStagedPiece(t, env, staged) + writeSourcePiece(t, env, source) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 1) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{staged.cidV2}, out.Pieces) + + stagedPP := parkedPieceByCID(t, env, staged) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, stagedPP.id), staged.data) + assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) + assertMissing(t, filepath.Join(env.stagingDir, staged.fileName)) +} + +func TestStorachaMigrationBatchProcessesFinalRecoveryBeforeSource(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + recovered := pieces.next() + source := pieces.next() + pieceID := insertParkedPiece(t, env, recovered, false, true, nil) + refID := insertStorachaParkedRef(t, env, pieceID) + insertPDPRef(t, env, refID, serviceName, recovered.info.PieceCIDV1.String()) + writeFinalPiece(t, env, pieceID, recovered.data) + writeSourcePiece(t, env, source) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 1) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{recovered.cidV2}, out.Pieces) + + pp := parkedPieceByID(t, env, pieceID) + require.True(t, pp.complete) + assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) +} + +func TestStorachaMigrationInvalidStagedFileDoesNotConsumeBatch(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + source := pieces.next() + invalidStagedPath := filepath.Join(env.stagingDir, "not-a-cid.car") + writeFile(t, invalidStagedPath, []byte("invalid staged input")) + writeSourcePiece(t, env, source) + + out, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 1) + require.NoError(t, err) + require.Equal(t, 1, out.Count) + require.Equal(t, []string{source.cidV2}, out.Pieces) + + pp := parkedPieceByCID(t, env, source) + assertFileBytes(t, storachaFinalPiecePath(env.targetDir, pp.id), source.data) + assertFileBytes(t, invalidStagedPath, []byte("invalid staged input")) +} + +func TestStorachaMigrationMissingSourcePathErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + missingSource := filepath.Join(filepath.Dir(env.sourceDir), "missing-source") + + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, missingSource, env.targetDir, 20) + require.ErrorContains(t, err, "reading directory") +} + +func TestStorachaMigrationSourceImportErrorsWhenStagingPathExists(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + source := pieces.next() + writeSourcePiece(t, env, source) + require.NoError(t, os.MkdirAll(filepath.Join(env.stagingDir, source.fileName), 0755)) + + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "staging file already exists") + assertFileBytes(t, filepath.Join(env.sourceDir, source.fileName), source.data) +} + +func TestStorachaMigrationStagedImportFinalDirectoryErrors(t *testing.T) { + env := setupStorachaMigrationTest(t) + pieces := newStorachaPieceFixtures(t) + fx := pieces.next() + pieceID := insertParkedPiece(t, env, fx, false, false, nil) + writeStagedPiece(t, env, fx) + require.NoError(t, os.MkdirAll(storachaFinalPiecePath(env.targetDir, pieceID), 0755)) + + _, err := runImportPieces(env.ctx, env.db, env.si, env.storageID, env.sourceDir, env.targetDir, 20) + require.ErrorContains(t, err, "final piece path is a directory") + + pp := parkedPieceByID(t, env, pieceID) + require.False(t, pp.complete) + assertFileBytes(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) + assertSectorLocation(t, env, pieceID, 0) +} + +func setupStorachaMigrationTest(t *testing.T) storachaMigrationTestEnv { + t.Helper() + + ctx := context.Background() + db, err := harmonydb.NewFromConfigWithITestID(t) + require.NoError(t, err) + + root := t.TempDir() + sourceDir := filepath.Join(root, "source") + storageDir := filepath.Join(root, "storage") + targetDir := storageDir + stagingDir := filepath.Join(targetDir, "storacha-staging") + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, storiface.FTPiece.String()), 0755)) + + storageID := storiface.ID(uuid.New().String()) + meta := &storiface.LocalStorageMeta{ + ID: storageID, + Weight: 10, + CanSeal: true, + CanStore: true, + } + mb, err := json.MarshalIndent(meta, "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(storageDir, paths.MetaFile), mb, 0644)) + + index := paths.NewDBIndex(nil, db) + + return storachaMigrationTestEnv{ + ctx: ctx, + db: db, + si: index, + storageID: storageID, + sourceDir: sourceDir, + targetDir: targetDir, + stagingDir: stagingDir, + } +} + +func newStorachaPieceFixtures(t *testing.T) *storachaPieceFixtures { + t.Helper() + return &storachaPieceFixtures{t: t} +} + +func (f *storachaPieceFixtures) next() storachaPieceFixture { + f.t.Helper() + f.seed++ + return newStorachaPieceFixture(f.t, f.seed) +} + +func newStorachaPieceFixture(t *testing.T, seed uint64) storachaPieceFixture { + t.Helper() + + rawSize := uint64(127) + var commP [32]byte + binary.BigEndian.PutUint64(commP[:8], seed) + binary.BigEndian.PutUint64(commP[8:16], rawSize) + pcidV2, err := commcid.DataCommitmentToPieceCidv2(commP[:], rawSize) + require.NoError(t, err) + + info, err := mk20.GetPieceInfo(pcidV2) + require.NoError(t, err) + + return storachaPieceFixture{ + cidV2: pcidV2.String(), + fileName: pcidV2.String() + ".car", + info: info, + data: bytes.Repeat([]byte{byte(seed % 251)}, int(rawSize)), + } +} + +func writeSourcePiece(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture) { + t.Helper() + writeFile(t, filepath.Join(env.sourceDir, fx.fileName), fx.data) +} + +func writeStagedPiece(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture) { + t.Helper() + writeFile(t, filepath.Join(env.stagingDir, fx.fileName), fx.data) +} + +func writeFinalPiece(t *testing.T, env storachaMigrationTestEnv, pieceID int64, data []byte) { + t.Helper() + writeFile(t, storachaFinalPiecePath(env.targetDir, pieceID), data) +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, data, 0644)) +} + +func assertFileBytes(t *testing.T, path string, expected []byte) { + t.Helper() + actual, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, expected, actual) +} + +func assertMissing(t *testing.T, path string) { + t.Helper() + _, err := os.Stat(path) + require.True(t, os.IsNotExist(err), "expected %s to be missing, got %v", path, err) +} + +func insertParkedPiece(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture, complete, skip bool, taskID *int64) int64 { + t.Helper() + + var pieceID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, complete, long_term, skip, task_id) + VALUES ($1, $2, $3, $4, TRUE, $5, $6) + RETURNING id + `, fx.info.PieceCIDV1.String(), int64(fx.info.Size), int64(fx.info.RawSize), complete, skip, taskID).Scan(&pieceID) + require.NoError(t, err) + return pieceID +} + +func insertStorachaParkedRef(t *testing.T, env storachaMigrationTestEnv, pieceID int64) int64 { + t.Helper() + + var refID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) + RETURNING ref_id + `, pieceID, storachaMigrationDataURL).Scan(&refID) + require.NoError(t, err) + return refID +} + +func insertPDPRef(t *testing.T, env storachaMigrationTestEnv, refID int64, service, pieceCID string) int64 { + t.Helper() + + var pdpID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO pdp_piecerefs (service, piece_cid, piece_ref, created_at, needs_save_cache) + VALUES ($1, $2, $3, NOW(), FALSE) + RETURNING id + `, service, pieceCID, refID).Scan(&pdpID) + require.NoError(t, err) + return pdpID +} + +func insertHarmonyTask(t *testing.T, env storachaMigrationTestEnv, name string) int64 { + t.Helper() + + var taskID int64 + err := env.db.QueryRow(env.ctx, ` + INSERT INTO harmony_task (name, added_by, posted_time) + VALUES ($1, 1, CURRENT_TIMESTAMP) + RETURNING id + `, name).Scan(&taskID) + require.NoError(t, err) + return taskID +} + +func parkedPieceByCID(t *testing.T, env storachaMigrationTestEnv, fx storachaPieceFixture) testParkedPiece { + t.Helper() + + var row testParkedPiece + err := env.db.QueryRow(env.ctx, ` + SELECT id, complete, skip, long_term, task_id + FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND long_term = TRUE + AND cleanup_task_id IS NULL + `, fx.info.PieceCIDV1.String(), int64(fx.info.Size)).Scan(&row.id, &row.complete, &row.skip, &row.longTerm, &row.taskID) + require.NoError(t, err) + return row +} + +func parkedPieceByID(t *testing.T, env storachaMigrationTestEnv, pieceID int64) testParkedPiece { + t.Helper() + + var row testParkedPiece + err := env.db.QueryRow(env.ctx, ` + SELECT id, complete, skip, long_term, task_id + FROM parked_pieces + WHERE id = $1 + `, pieceID).Scan(&row.id, &row.complete, &row.skip, &row.longTerm, &row.taskID) + require.NoError(t, err) + return row +} + +func storachaRefIDs(t *testing.T, env storachaMigrationTestEnv, pieceID int64) []int64 { + t.Helper() + return storachaRefIDsForDataURL(t, env, pieceID, storachaMigrationDataURL) +} + +func storachaRefIDsForDataURL(t *testing.T, env storachaMigrationTestEnv, pieceID int64, dataURL string) []int64 { + t.Helper() + + var rows []struct { + RefID int64 `db:"ref_id"` + } + err := env.db.Select(env.ctx, &rows, ` + SELECT ref_id + FROM parked_piece_refs + WHERE piece_id = $1 + AND data_url = $2 + AND long_term = TRUE + ORDER BY ref_id + `, pieceID, dataURL) + require.NoError(t, err) + + out := make([]int64, 0, len(rows)) + for _, row := range rows { + out = append(out, row.RefID) + } + return out +} + +func pdpRefsForPiece(t *testing.T, env storachaMigrationTestEnv, pieceID int64) []testPDPRef { + t.Helper() + + var rows []testPDPRef + err := env.db.Select(env.ctx, &rows, ` + SELECT p.id, p.service, p.piece_cid, p.piece_ref + FROM pdp_piecerefs p + JOIN parked_piece_refs r ON r.ref_id = p.piece_ref + WHERE r.piece_id = $1 + ORDER BY p.id + `, pieceID) + require.NoError(t, err) + return rows +} + +func assertStorachaRefs(t *testing.T, env storachaMigrationTestEnv, pieceID int64, expected int) { + t.Helper() + require.Len(t, storachaRefIDs(t, env, pieceID), expected) +} + +func assertStorachaRefsWithDataURL(t *testing.T, env storachaMigrationTestEnv, pieceID int64, dataURL string, expected int) { + t.Helper() + require.Len(t, storachaRefIDsForDataURL(t, env, pieceID, dataURL), expected) +} + +func assertPDPRefs(t *testing.T, env storachaMigrationTestEnv, pieceID int64, pieceCID string, expected int) { + t.Helper() + refs := pdpRefsForPiece(t, env, pieceID) + require.Len(t, refs, expected) + for _, ref := range refs { + require.Equal(t, serviceName, ref.Service) + require.Equal(t, pieceCID, ref.PieceCID) + } +} + +func assertSectorLocation(t *testing.T, env storachaMigrationTestEnv, pieceID int64, expected int) { + t.Helper() + + var count int + err := env.db.QueryRow(env.ctx, ` + SELECT COUNT(*) + FROM sector_location + WHERE miner_id = 0 + AND sector_num = $1 + AND sector_filetype = $2 + AND storage_id = $3 + `, pieceID, int(storiface.FTPiece), string(env.storageID)).Scan(&count) + require.NoError(t, err) + require.Equal(t, expected, count) +} diff --git a/cmd/curio/toolbox.go b/cmd/curio/toolbox.go index d478a9f3d..9488b4327 100644 --- a/cmd/curio/toolbox.go +++ b/cmd/curio/toolbox.go @@ -40,6 +40,8 @@ var toolboxCmd = &cli.Command{ fixMsgCmd, downgradeCmd, fixBoostMigrationCmd, + importPiecesCmd, + aggregatePiecesCmd, }, } diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index 52dfa0d63..a077f0765 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -1384,6 +1384,8 @@ COMMANDS: fix-msg Updated DB with message data missing from chain node downgrade Downgrade a cluster's database to a previous software version. fix-boost-migration Fix Boost migration + import-pieces Imports already existing pieces from storage to piece park system + aggregate-pieces Aggregates already existing pieces from storage into PODSI and park them help, h Shows a list of commands or help for one command OPTIONS: @@ -1436,6 +1438,38 @@ OPTIONS: --help, -h show help ``` +### curio toolbox import-pieces +``` +NAME: + curio toolbox import-pieces - Imports already existing pieces from storage to piece park system + +USAGE: + curio toolbox import-pieces [command options] + +OPTIONS: + --source value path to store the pieces from + --target value path to storage directory in Curio's attached permanent storage + --result value path to write the JSON result + --batch-size value number of pieces to move to permanent storage (default: 20) + --help, -h show help +``` + +### curio toolbox aggregate-pieces +``` +NAME: + curio toolbox aggregate-pieces - Aggregates already existing pieces from storage into PODSI and park them + +USAGE: + curio toolbox aggregate-pieces [command options] + +OPTIONS: + --source value path to read the pieces from + --target value path to storage directory in Curio's attached permanent storage + --result value path to write the JSON result + --input value path to read the CIDs from + --help, -h show help +``` + ## curio batch ``` NAME: diff --git a/tasks/indexing/task_pdp_v0_ipni.go b/tasks/indexing/task_pdp_v0_ipni.go index 5e9a61993..faa915af7 100644 --- a/tasks/indexing/task_pdp_v0_ipni.go +++ b/tasks/indexing/task_pdp_v0_ipni.go @@ -317,10 +317,10 @@ func (P *PDPV0IPNITask) TypeDetails() harmonytask.TaskTypeDetails { Ram: 1 << 30, }, MaxFailures: 3, - IAmBored: passcall.Every(30*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { return P.schedule(context.Background(), taskFunc) }), - Max: P.max, + Max: taskhelp.Max(50), } }