Fix ext4 offset (#313)

* Implement sub-storage backend for reading from an offset filesystem at any point within an img file
* Create a second ext4 test image with offset, along with tests for offset ext4 img

Author: StoneMonarch

backend/substorage.go

@@ -0,0 +1,127 @@
+package backend
+
+import (
+	"io"
+	"io/fs"
+	"os"
+)
+
+type SubStorage struct {
+	underlying Storage
+	offset     int64
+	size       int64
+}
+
+func Sub(u Storage, offset, size int64) Storage {
+	return SubStorage{
+		underlying: u,
+		offset:     offset,
+		size:       size,
+	}
+}
+
+func (s SubStorage) Stat() (fs.FileInfo, error) {
+	return s.underlying.Stat()
+}
+
+func (s SubStorage) Read(bytes []byte) (int, error) {
+	return s.underlying.Read(bytes)
+}
+
+func (s SubStorage) Close() error {
+	return s.underlying.Close()
+}
+
+func (s SubStorage) ReadAt(p []byte, off int64) (n int, err error) {
+	return s.underlying.ReadAt(p, s.offset+off)
+}
+
+func (s SubStorage) Seek(offset int64, whence int) (int64, error) {
+	var (
+		pos int64
+		err error
+	)
+
+	switch whence {
+	case io.SeekStart:
+		pos, err = s.underlying.Seek(offset+s.offset, io.SeekStart)
+	case io.SeekCurrent:
+		pos, err = s.underlying.Seek(offset, io.SeekCurrent)
+	case io.SeekEnd:
+		pos, err = s.underlying.Seek(s.offset+s.size+offset, io.SeekStart)
+	default:
+		return -1, ErrNotSuitable
+	}
+
+	if err != nil {
+		return -1, err
+	}
+
+	return pos - s.offset, nil
+}
+
+func (s SubStorage) Sys() (*os.File, error) {
+	return s.underlying.Sys()
+}
+
+func (s SubStorage) Writable() (WritableFile, error) {
+	uw, err := s.underlying.Writable()
+	if err != nil {
+		return nil, err
+	}
+	return subWritable{
+		underlying: uw,
+		offset:     s.offset,
+		size:       s.size,
+	}, nil
+}
+
+type subWritable struct {
+	underlying WritableFile
+	offset     int64
+	size       int64
+}
+
+func (sw subWritable) Stat() (fs.FileInfo, error) {
+	return sw.underlying.Stat()
+}
+
+func (sw subWritable) Read(b []byte) (int, error) {
+	return sw.underlying.Read(b)
+}
+
+func (sw subWritable) Close() error {
+	return sw.underlying.Close()
+}
+
+func (sw subWritable) ReadAt(p []byte, off int64) (n int, err error) {
+	return sw.underlying.ReadAt(p, sw.offset+off)
+}
+
+func (sw subWritable) Seek(offset int64, whence int) (int64, error) {
+	var (
+		pos int64
+		err error
+	)
+
+	switch whence {
+	case io.SeekStart:
+		pos, err = sw.underlying.Seek(offset+sw.offset, io.SeekStart)
+	case io.SeekCurrent:
+		pos, err = sw.underlying.Seek(offset, io.SeekCurrent)
+	case io.SeekEnd:
+		pos, err = sw.underlying.Seek(sw.offset+sw.size+offset, io.SeekStart)
+	default:
+		return -1, ErrNotSuitable
+	}
+
+	if err != nil {
+		return -1, err
+	}
+
+	return pos - sw.offset, nil
+}
+
+func (sw subWritable) WriteAt(p []byte, off int64) (n int, err error) {
+	return sw.underlying.WriteAt(p, sw.offset+off)
+}

filesystem/ext4/common_test.go

@@ -18,6 +18,7 @@ import (
 
 const (
 	imgFile             = "testdata/dist/ext4.img"
+	imgFileOffset       = "testdata/dist/ext4-offset.img"
 	fooDirFile          = "testdata/dist/foo_dir.txt"
 	testGDTFile         = "testdata/dist/gdt.bin"
 	rootDirFile         = "testdata/dist/root_dir.txt"
@@ -30,7 +31,14 @@ const (
 // TestMain sets up the test environment and runs the tests
 func TestMain(m *testing.M) {
 	// Check and generate artifacts if necessary
+	needGen := false
 	if _, err := os.Stat(imgFile); os.IsNotExist(err) {
+		needGen = true
+	}
+	if _, err := os.Stat(imgFileOffset); os.IsNotExist(err) {
+		needGen = true
+	}
+	if needGen {
 		// Run the genartifacts.sh script
 		cmd := exec.Command("sh", "buildimg.sh")
 		cmd.Stdout = os.Stdout

filesystem/ext4/ext4.go

@@ -559,6 +559,10 @@ func Create(b backend.Storage, size, start, sectorsize int64, p *Params) (*FileS
 	groupCount := sb.blockGroupCount()
 	gdtSize := uint64(gdSize) * groupCount
 	// write the superblock and GDT to the various locations on disk
+
+	// Make SubStorage Backend
+	fsBackend := backend.Sub(b, start, size)
+
 	for _, bg := range backupSuperblocks {
 		block := bg * int64(blocksPerGroup)
 		blockStart := block * int64(blocksize)
@@ -568,13 +572,13 @@ func Create(b backend.Storage, size, start, sectorsize int64, p *Params) (*FileS
 			incr = int64(SectorSize512) * 2
 		}
 
-		writable, err := b.Writable()
+		writable, err := fsBackend.Writable()
 		if err != nil {
 			return nil, err
 		}
 
 		// write the superblock
-		count, err := writable.WriteAt(superblockBytes, incr+blockStart+start)
+		count, err := writable.WriteAt(superblockBytes, incr+blockStart)
 		if err != nil {
 			return nil, fmt.Errorf("error writing Superblock for block %d to disk: %v", block, err)
 		}
@@ -583,7 +587,7 @@ func Create(b backend.Storage, size, start, sectorsize int64, p *Params) (*FileS
 		}
 
 		// write the GDT
-		count, err = writable.WriteAt(g, incr+blockStart+int64(SuperblockSize)+start)
+		count, err = writable.WriteAt(g, incr+blockStart+int64(SuperblockSize))
 		if err != nil {
 			return nil, fmt.Errorf("error writing GDT for block %d to disk: %v", block, err)
 		}
@@ -601,7 +605,7 @@ func Create(b backend.Storage, size, start, sectorsize int64, p *Params) (*FileS
 		blockGroups:      blockGroups,
 		size:             size,
 		start:            start,
-		backend:          b,
+		backend:          fsBackend,
 	}, nil
 }
 
@@ -631,10 +635,13 @@ func Read(b backend.Storage, size, start, sectorsize int64) (*FileSystem, error)
 		return nil, fmt.Errorf("requested size is smaller than minimum allowed ext4 size %d", Ext4MinSize)
 	}
 
+	// Make SubStorage Backend
+	fsBackend := backend.Sub(b, start, size)
+
 	// load the information from the disk
 	// read boot sector code
 	bs := make([]byte, BootSectorSize)
-	n, err := b.ReadAt(bs, start)
+	n, err := fsBackend.ReadAt(bs, 0)
 	if err != nil {
 		return nil, fmt.Errorf("could not read boot sector bytes from file: %v", err)
 	}
@@ -645,7 +652,7 @@ func Read(b backend.Storage, size, start, sectorsize int64) (*FileSystem, error)
 	// read the superblock
 	// the superblock is one minimal block, i.e. 2 sectors
 	superblockBytes := make([]byte, SuperblockSize)
-	n, err = b.ReadAt(superblockBytes, start+int64(BootSectorSize))
+	n, err = fsBackend.ReadAt(superblockBytes, int64(BootSectorSize))
 	if err != nil {
 		return nil, fmt.Errorf("could not read superblock bytes from file: %v", err)
 	}
@@ -677,7 +684,7 @@ func Read(b backend.Storage, size, start, sectorsize int64) (*FileSystem, error)
 	if sb.blockSize == 1024 {
 		gdtBlock = 2
 	}
-	n, err = b.ReadAt(gdtBytes, start+int64(gdtBlock)*int64(sb.blockSize))
+	n, err = fsBackend.ReadAt(gdtBytes, int64(gdtBlock)*int64(sb.blockSize))
 	if err != nil {
 		return nil, fmt.Errorf("could not read Group Descriptor Table bytes from file: %v", err)
 	}
@@ -696,7 +703,7 @@ func Read(b backend.Storage, size, start, sectorsize int64) (*FileSystem, error)
 		blockGroups:      int64(sb.blockGroupCount()),
 		size:             size,
 		start:            start,
-		backend:          b,
+		backend:          fsBackend,
 	}, nil
 }
 
@@ -969,7 +976,7 @@ func (fs *FileSystem) Remove(p string) error {
 		gd.freeBlocks += freedByBG[bg]
 		gd.blockBitmapChecksum = bitmapChecksum(dataBlockBitmap.ToBytes(), fs.superblock.checksumSeed)
 		gdBytes := gd.toBytes(fs.superblock.gdtChecksumType(), fs.superblock.checksumSeed)
-		if _, err := writableFile.WriteAt(gdBytes, fs.start+int64(gdtBlock)*int64(fs.superblock.blockSize)+int64(gd.number)*int64(fs.superblock.groupDescriptorSize)); err != nil {
+		if _, err := writableFile.WriteAt(gdBytes, int64(gdtBlock)*int64(fs.superblock.blockSize)+int64(gd.number)*int64(fs.superblock.groupDescriptorSize)); err != nil {
 			return fmt.Errorf("could not write Group Descriptor bytes to file: %v", err)
 		}
 	}
@@ -1011,7 +1018,7 @@ func (fs *FileSystem) Remove(p string) error {
 			}
 			b := dirBytes[start:end]
 
-			fileOff := fs.start + (int64(e.startingBlock)+int64(i))*int64(bs)
+			fileOff := (int64(e.startingBlock) + int64(i)) * int64(bs)
 
 			if _, err := writableFile.WriteAt(b, fileOff); err != nil {
 				return fmt.Errorf("could not write inode bitmap back to disk: %v", err)
@@ -1058,7 +1065,7 @@ func (fs *FileSystem) Remove(p string) error {
 
 	// write the group descriptor back
 	gdBytes := gd.toBytes(fs.superblock.gdtChecksumType(), fs.superblock.checksumSeed)
-	if _, err := writableFile.WriteAt(gdBytes, fs.start+int64(gdtBlock)*int64(fs.superblock.blockSize)+int64(gd.number)*int64(fs.superblock.groupDescriptorSize)); err != nil {
+	if _, err := writableFile.WriteAt(gdBytes, int64(gdtBlock)*int64(fs.superblock.blockSize)+int64(gd.number)*int64(fs.superblock.groupDescriptorSize)); err != nil {
 		return fmt.Errorf("could not write Group Descriptor bytes to file: %v", err)
 	}
 
@@ -1650,7 +1657,7 @@ func (fs *FileSystem) allocateInode(parent uint32) (uint32, error) {
 	// gdt starts in block 1 of any redundant copies, specifically in BG 0
 	gdtBlock := 1
 	blockByteLocation := gdtBlock * int(fs.superblock.blockSize)
-	gdOffset := fs.start + int64(blockByteLocation) + int64(bg)*int64(fs.superblock.groupDescriptorSize)
+	gdOffset := int64(blockByteLocation) + int64(bg)*int64(fs.superblock.groupDescriptorSize)
 	wrote, err := writableFile.WriteAt(gdBytes, gdOffset)
 	if err != nil {
 		return 0, fmt.Errorf("unable to write group descriptor bytes for blockgroup %d: %v", bg, err)
@@ -1802,7 +1809,7 @@ func (fs *FileSystem) readInodeBitmap(group int) (*bitmap.Bitmap, error) {
 	bitmapLocation := gd.inodeBitmapLocation
 	bitmapByteCount := fs.superblock.inodesPerGroup / 8
 	b := make([]byte, bitmapByteCount)
-	offset := int64(bitmapLocation*uint64(fs.superblock.blockSize) + uint64(fs.start))
+	offset := int64(bitmapLocation * uint64(fs.superblock.blockSize))
 	read, err := fs.backend.ReadAt(b, offset)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read inode bitmap for blockgroup %d: %w", gd.number, err)
@@ -1831,7 +1838,7 @@ func (fs *FileSystem) writeInodeBitmap(bm *bitmap.Bitmap, group int) error {
 	gd := fs.groupDescriptors.descriptors[group]
 	bitmapByteCount := fs.superblock.inodesPerGroup / 8
 	bitmapLocation := gd.inodeBitmapLocation
-	offset := int64(bitmapLocation*uint64(fs.superblock.blockSize) + uint64(fs.start))
+	offset := int64(bitmapLocation * uint64(fs.superblock.blockSize))
 	wrote, err := writableFile.WriteAt(b, offset)
 	if err != nil {
 		return fmt.Errorf("unable to write inode bitmap for blockgroup %d: %w", gd.number, err)
@@ -1850,7 +1857,7 @@ func (fs *FileSystem) readBlockBitmap(group int) (*bitmap.Bitmap, error) {
 	gd := fs.groupDescriptors.descriptors[group]
 	bitmapLocation := gd.blockBitmapLocation
 	b := make([]byte, fs.superblock.blockSize)
-	offset := int64(bitmapLocation*uint64(fs.superblock.blockSize) + uint64(fs.start))
+	offset := int64(bitmapLocation * uint64(fs.superblock.blockSize))
 	read, err := fs.backend.ReadAt(b, offset)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read block bitmap for blockgroup %d: %w", gd.number, err)
@@ -1876,7 +1883,7 @@ func (fs *FileSystem) writeBlockBitmap(bm *bitmap.Bitmap, group int) error {
 	b := bm.ToBytes()
 	gd := fs.groupDescriptors.descriptors[group]
 	bitmapLocation := gd.blockBitmapLocation
-	offset := int64(bitmapLocation*uint64(fs.superblock.blockSize) + uint64(fs.start))
+	offset := int64(bitmapLocation * uint64(fs.superblock.blockSize))
 	wrote, err := writableFile.WriteAt(b, offset)
 	if err != nil {
 		return fmt.Errorf("unable to write block bitmap for blockgroup %d: %w", gd.number, err)
@@ -1897,7 +1904,7 @@ func (fs *FileSystem) writeSuperblock() error {
 	if err != nil {
 		return fmt.Errorf("could not convert superblock to bytes: %v", err)
 	}
-	_, err = writableFile.WriteAt(superblockBytes, fs.start+int64(BootSectorSize))
+	_, err = writableFile.WriteAt(superblockBytes, int64(BootSectorSize))
 	return err
 }