-
-
Notifications
You must be signed in to change notification settings - Fork 104
refactor(reflinking): add windows ReFS filesystem support #1576
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
ff9bffd
refactor(reflinking): add windows ReFS filesystem support
Audionut 3dfdddd
fix(buffer): use pooling
Audionut 4c68a33
fix(reflinktree): ensure Windows probe exercises block clone path
Audionut 01f3126
fix(reflinktree): stat Windows source file from opened handle
Audionut 32bde1e
fix(reflinktree): handle Windows symlink and sparse ReFS sources
Audionut b6f2d35
fix(reflink): handle Windows unsupported clone paths and resolved des…
Audionut 7cf1b0e
fix(crossseed): keep rollback-wrapped reflink errors at error level
Audionut 0fadd0d
Merge branch 'develop' into feat/windows-reflink-support
Audionut 0a3a670
docs(crossseed): clarify Windows reflink symlink handling
Audionut 5173173
fix(crossseed): handle joined reflink errors
s0up4200 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,299 @@ | ||
| // Copyright (c) 2025-2026, s0up and the autobrr contributors. | ||
| // SPDX-License-Identifier: GPL-2.0-or-later | ||
|
|
||
| //go:build windows | ||
|
|
||
| package reflinktree | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "unsafe" | ||
|
|
||
| "golang.org/x/sys/windows" | ||
| ) | ||
|
|
||
| const ( | ||
| fsctlDuplicateExtentsToFile = 0x00098344 | ||
| maxCloneChunkSize = 1024 * 1024 * 1024 | ||
| copyBufferSize = 1024 * 1024 | ||
| refsFilesystemName = "REFS" | ||
| ) | ||
|
|
||
| type duplicateExtentsData struct { | ||
| FileHandle windows.Handle | ||
| SourceFileOffset int64 | ||
| TargetFileOffset int64 | ||
| ByteCount int64 | ||
| } | ||
|
|
||
| var ( | ||
| kernel32DLL = windows.NewLazySystemDLL("kernel32.dll") | ||
| procGetDiskFreeSpaceW = kernel32DLL.NewProc("GetDiskFreeSpaceW") | ||
| volumeRootForPathFn = getVolumeRoot | ||
| filesystemNameForVolFn = getFilesystemName | ||
| clusterSizeForVolFn = getClusterSize | ||
| duplicateExtentFn = duplicateExtent | ||
| copyFileTailFn = copyFileTail | ||
| ) | ||
|
|
||
| // SupportsReflink tests whether the given directory supports reflinks | ||
| // by attempting an actual clone operation with temporary files. | ||
| // Returns true if reflinks are supported, along with a reason string. | ||
| func SupportsReflink(dir string) (supported bool, reason string) { | ||
| if err := os.MkdirAll(dir, 0o755); err != nil { | ||
| return false, fmt.Sprintf("cannot access directory: %v", err) | ||
| } | ||
|
|
||
| srcFile, err := os.CreateTemp(dir, ".reflink_probe_src_*") | ||
| if err != nil { | ||
| return false, fmt.Sprintf("cannot create temp file: %v", err) | ||
| } | ||
| srcPath := srcFile.Name() | ||
| defer os.Remove(srcPath) | ||
|
|
||
| if _, err := srcFile.WriteString("reflink probe test data"); err != nil { | ||
| srcFile.Close() | ||
| return false, fmt.Sprintf("cannot write to temp file: %v", err) | ||
| } | ||
| if err := srcFile.Close(); err != nil { | ||
| return false, fmt.Sprintf("cannot close temp file: %v", err) | ||
| } | ||
|
|
||
| dstPath := filepath.Join(dir, ".reflink_probe_dst_"+filepath.Base(srcPath)[len(".reflink_probe_src_"):]) | ||
| defer os.Remove(dstPath) | ||
|
|
||
| if err := cloneFile(srcPath, dstPath); err != nil { | ||
| return false, fmt.Sprintf("reflink not supported: %v", err) | ||
| } | ||
|
|
||
| return true, "reflink supported (ReFS block cloning)" | ||
| } | ||
|
|
||
| func cloneFile(src, dst string) (retErr error) { | ||
| srcInfo, err := os.Stat(src) | ||
| if err != nil { | ||
| return fmt.Errorf("stat source: %w", err) | ||
| } | ||
|
|
||
| dstParent := filepath.Dir(dst) | ||
| if dstParent == "" { | ||
| dstParent = "." | ||
| } | ||
|
|
||
| volumeRoot, err := ensureSameVolume(src, dstParent) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| clusterSize, err := ensureRefsVolume(volumeRoot) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| srcFile, err := os.Open(src) | ||
| if err != nil { | ||
| return fmt.Errorf("open source: %w", err) | ||
| } | ||
coderabbitai[bot] marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| defer srcFile.Close() | ||
|
|
||
| dstFile, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, srcInfo.Mode()) | ||
| if err != nil { | ||
| return fmt.Errorf("create destination: %w", err) | ||
| } | ||
| defer func() { | ||
| _ = dstFile.Close() | ||
| if retErr != nil { | ||
| _ = os.Remove(dst) | ||
| } | ||
| }() | ||
|
|
||
| if err := dstFile.Truncate(srcInfo.Size()); err != nil { | ||
| return fmt.Errorf("resize destination: %w", err) | ||
| } | ||
|
|
||
| srcHandle := windows.Handle(srcFile.Fd()) | ||
| dstHandle := windows.Handle(dstFile.Fd()) | ||
| cloneableSize := srcInfo.Size() - (srcInfo.Size() % clusterSize) | ||
|
|
||
| for offset := int64(0); offset < cloneableSize; offset += maxCloneChunkSize { | ||
| chunkSize := min(maxCloneChunkSize, cloneableSize-offset) | ||
| if err := duplicateExtentFn(dstHandle, srcHandle, offset, offset, chunkSize); err != nil { | ||
| return fmt.Errorf("duplicate extents: %w", err) | ||
| } | ||
| } | ||
|
|
||
| if tailSize := srcInfo.Size() - cloneableSize; tailSize > 0 { | ||
| if err := copyFileTailFn(srcFile, dstFile, cloneableSize, tailSize); err != nil { | ||
| return fmt.Errorf("copy file tail: %w", err) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func ensureSameVolume(src, dst string) (string, error) { | ||
| srcRoot, err := volumeRootForPathFn(src) | ||
| if err != nil { | ||
| return "", fmt.Errorf("get source volume: %w", err) | ||
| } | ||
|
|
||
| dstRoot, err := volumeRootForPathFn(dst) | ||
| if err != nil { | ||
| return "", fmt.Errorf("get destination volume: %w", err) | ||
| } | ||
|
|
||
| if !strings.EqualFold(srcRoot, dstRoot) { | ||
| return "", errors.New("source and destination must be on the same volume") | ||
| } | ||
|
|
||
| return srcRoot, nil | ||
| } | ||
|
|
||
| func ensureRefsVolume(volumeRoot string) (int64, error) { | ||
| filesystemName, err := filesystemNameForVolFn(volumeRoot) | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
|
|
||
| if !strings.EqualFold(filesystemName, refsFilesystemName) { | ||
| return 0, fmt.Errorf("volume %s is %s, not ReFS", volumeRoot, filesystemName) | ||
| } | ||
|
|
||
| clusterSize, err := clusterSizeForVolFn(volumeRoot) | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| if clusterSize <= 0 { | ||
| return 0, errors.New("invalid cluster size") | ||
| } | ||
|
|
||
| return clusterSize, nil | ||
| } | ||
|
|
||
| func getVolumeRoot(path string) (string, error) { | ||
| absPath, err := filepath.Abs(path) | ||
| if err != nil { | ||
| return "", fmt.Errorf("abs path: %w", err) | ||
| } | ||
|
|
||
| volumePath := make([]uint16, windows.MAX_PATH+1) | ||
| pathPtr, err := windows.UTF16PtrFromString(absPath) | ||
| if err != nil { | ||
| return "", fmt.Errorf("convert path: %w", err) | ||
| } | ||
|
|
||
| if err := windows.GetVolumePathName(pathPtr, &volumePath[0], uint32(len(volumePath))); err != nil { | ||
| return "", fmt.Errorf("get volume path name: %w", err) | ||
| } | ||
|
|
||
| volumeRoot := windows.UTF16ToString(volumePath) | ||
| if !strings.HasSuffix(volumeRoot, `\`) { | ||
| volumeRoot += `\` | ||
| } | ||
|
|
||
| return volumeRoot, nil | ||
| } | ||
|
|
||
| func getFilesystemName(volumeRoot string) (string, error) { | ||
| volumePathPtr, err := windows.UTF16PtrFromString(volumeRoot) | ||
| if err != nil { | ||
| return "", fmt.Errorf("convert volume path: %w", err) | ||
| } | ||
|
|
||
| filesystemName := make([]uint16, windows.MAX_PATH+1) | ||
| var volumeSerial uint32 | ||
| var maxComponentLength uint32 | ||
| var flags uint32 | ||
| if err := windows.GetVolumeInformation( | ||
| volumePathPtr, | ||
| nil, | ||
| 0, | ||
| &volumeSerial, | ||
| &maxComponentLength, | ||
| &flags, | ||
| &filesystemName[0], | ||
| uint32(len(filesystemName)), | ||
| ); err != nil { | ||
| return "", fmt.Errorf("get volume information: %w", err) | ||
| } | ||
|
|
||
| name := windows.UTF16ToString(filesystemName) | ||
| if name == "" { | ||
| return "", errors.New("filesystem name is empty") | ||
| } | ||
|
|
||
| return name, nil | ||
| } | ||
|
|
||
| func getClusterSize(volumeRoot string) (int64, error) { | ||
| volumePathPtr, err := windows.UTF16PtrFromString(volumeRoot) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("convert volume path: %w", err) | ||
| } | ||
|
|
||
| var sectorsPerCluster uint32 | ||
| var bytesPerSector uint32 | ||
| var freeClusters uint32 | ||
| var totalClusters uint32 | ||
| r1, _, callErr := procGetDiskFreeSpaceW.Call( | ||
| uintptr(unsafe.Pointer(volumePathPtr)), | ||
| uintptr(unsafe.Pointer(§orsPerCluster)), | ||
| uintptr(unsafe.Pointer(&bytesPerSector)), | ||
| uintptr(unsafe.Pointer(&freeClusters)), | ||
| uintptr(unsafe.Pointer(&totalClusters)), | ||
| ) | ||
| if r1 == 0 { | ||
| if callErr != nil && !errors.Is(callErr, windows.ERROR_SUCCESS) { | ||
| return 0, fmt.Errorf("get cluster size: %w", callErr) | ||
| } | ||
| return 0, errors.New("get cluster size: unknown error") | ||
| } | ||
|
|
||
| return int64(sectorsPerCluster) * int64(bytesPerSector), nil | ||
| } | ||
|
|
||
| func duplicateExtent(targetHandle, sourceHandle windows.Handle, sourceOffset, targetOffset, byteCount int64) error { | ||
| data := duplicateExtentsData{ | ||
| FileHandle: sourceHandle, | ||
| SourceFileOffset: sourceOffset, | ||
| TargetFileOffset: targetOffset, | ||
| ByteCount: byteCount, | ||
| } | ||
|
|
||
| var bytesReturned uint32 | ||
| return windows.DeviceIoControl( | ||
| targetHandle, | ||
| fsctlDuplicateExtentsToFile, | ||
| (*byte)(unsafe.Pointer(&data)), | ||
| uint32(unsafe.Sizeof(data)), | ||
| nil, | ||
| 0, | ||
| &bytesReturned, | ||
| nil, | ||
| ) | ||
| } | ||
|
|
||
| func copyFileTail(srcFile, dstFile *os.File, offset, length int64) error { | ||
| if _, err := srcFile.Seek(offset, io.SeekStart); err != nil { | ||
| return fmt.Errorf("seek source: %w", err) | ||
| } | ||
| if _, err := dstFile.Seek(offset, io.SeekStart); err != nil { | ||
| return fmt.Errorf("seek destination: %w", err) | ||
| } | ||
|
|
||
| buffer := make([]byte, copyBufferSize) | ||
| copied, err := io.CopyBuffer(dstFile, io.LimitReader(srcFile, length), buffer) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if copied != length { | ||
| return io.ErrUnexpectedEOF | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.