filestore: Allow customising the storage path (#1083)

* filestore: Allow customising the destination path

* Compatible with uploads from previous version

* Add todo about documentation

* More comments

* Documentation

Author: Marius

docs/_advanced-topics/hooks.md

@@ -82,7 +82,6 @@ Below you can find an annotated, JSON-ish encoded example of a hook request:
             "PartialUploads": null,
             // Storage contains information about where the upload is stored. The exact values
             // depend on the storage that is used and are not available in the pre-create hook.
-            // This example belongs to the file store. 
             "Storage": {
                  // For example, the filestore supplies the absolute file path:
                  "Type": "filestore",
@@ -190,6 +189,28 @@ Below you can find an annotated, JSON-ish encoded example of a hook response:
         // in the Upload-Metadata header in HEAD responses.
         "MetaData": {
           "my-custom-field": "..."
+        },
+        // Storage can be used to customize the location where the uploaded file (aka the binary
+        // file is saved). The exact behavior depends on the storage that is used. Please note
+        // that this only influences the location of the binary file. tusd will still create an
+        // info file whose location is derived from the upload ID and can not be customized using
+        // this ChangeFileInfo.Storage property, but only using ChangeFileInfo.ID.
+        //
+        // The location can contain forward slashes (/) to store uploads in a hierarchical structure,
+        // such as nested directories.
+        //
+        // Similar to ChangeFileInfo.ID, tusd will not check whether a file is already saved under
+        // this location and might overwrite it. It is the hooks responsibility to ensure that
+        // the location is save to use. A good approach is to embed a random part (e.g. a UUID) in
+        // the location.
+        "Storage": {
+            // When the filestore is used, the Path property defines where the uploaded file is saved.
+            // The path may be absolute or relative, and point towards a location outside of the directory
+            // defined using the `-dir` flag. If it's relative, the path will be resolved relative to `-dir`.
+            "Path": "./upload-e7a036dc-33f4-451f-9520-49032b87e952/presentation.pdf"
+
+            // Other storages, such as S3Store, GCSStore, and AzureStore, do not support the Storage
+            // property yet.
         }
     },
 

docs/_storage-backends/local-disk.md

@@ -17,6 +17,25 @@ When a new upload is created, for example with the upload ID `abcdef123`, tusd c
 - `./uploads/abcdef123` to hold the file content that is uploaded
 - `./uploads/abcdef123.info` to hold [meta information about the upload]({{ site.baseurl }}/storage-backends/overview/#storage-format)
 
+## Custom storage location
+
+The locations of the two files mentioned above can be fully customized using the [pre-create hook](({ site.baseurl }}/advanced-topics/hooks/). The location of the `.info` file is derived from the upload ID, which can be customized by the pre-create hook using the [`ChangeFileInfo.ID` setting]({ site.baseurl }}/advanced-topics/hooks/#hook-requests-and-responses). Similarly, the location where the file content is saved is by default derived from the upload ID, but can be fully customized using the [`ChangeFileInfo.Storage.Path` setting]({ site.baseurl }}/advanced-topics/hooks/#hook-requests-and-responses).
+
+For example, if the pre-create hook returns the following hook response, an upload with ID `project-123/abc` is created, the info file is saved at `./uploads/project-123/abc.info`, and the file content is saved at `./uploads/project-123/abc/presentation.pdf`:
+
+```js
+{
+    "ChangeFileInfo": {
+        "ID": "project-123/abc",
+        "Storage": {
+            "Path": "project-123/abc/presentation.pdf"
+        }
+    },
+}
+```
+
+If the defined path is relative, it will be resolved from the directory defined using `-dir`.
+
 ## Issues with NFS and shared folders
 
 Tusd maintains [upload locks]({{ site.baseurl }}/advanced-topics/locks/) on disk to ensure exclusive access to uploads and prevent data corruption. These disk-based locks utilize hard links, which might not be supported by older NFS versions or when a folder is shared in a VM using VirtualBox or Vagrant. In these cases, you might get errors like this:

pkg/filestore/filestore.go

@@ -56,7 +56,17 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha
 	if info.ID == "" {
 		info.ID = uid.Uid()
 	}
-	binPath := store.binPath(info.ID)
+
+	// The .info file's location can directly be deduced from the upload ID
+	infoPath := store.infoPath(info.ID)
+	// The binary file's location might be modified by the pre-create hook.
+	var binPath string
+	if info.Storage != nil && info.Storage["Path"] != "" {
+		binPath = filepath.Join(store.Path, info.Storage["Path"])
+	} else {
+		binPath = store.defaultBinPath(info.ID)
+	}
+
 	info.Storage = map[string]string{
 		"Type": "filestore",
 		"Path": binPath,
@@ -69,7 +79,7 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha
 
 	upload := &fileUpload{
 		info:     info,
-		infoPath: store.infoPath(info.ID),
+		infoPath: infoPath,
 		binPath:  binPath,
 	}
 
@@ -82,21 +92,32 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha
 }
 
 func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) {
-	info := handler.FileInfo{}
-	data, err := os.ReadFile(store.infoPath(id))
+	infoPath := store.infoPath(id)
+	data, err := os.ReadFile(infoPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// Interpret os.ErrNotExist as 404 Not Found
 			err = handler.ErrNotFound
 		}
 		return nil, err
 	}
+	var info handler.FileInfo
 	if err := json.Unmarshal(data, &info); err != nil {
 		return nil, err
 	}
 
-	binPath := store.binPath(id)
-	infoPath := store.infoPath(id)
+	// If the info file contains a custom path to the binary file, we use that. If not, we
+	// fall back to the default value (although the Path property should always be set in recent
+	// tusd versions).
+	var binPath string
+	if info.Storage != nil && info.Storage["Path"] != "" {
+		// No filepath.Join here because the joining already happened in NewUpload. Duplicate joining
+		// with relative paths lead to incorrect paths
+		binPath = info.Storage["Path"]
+	} else {
+		binPath = store.defaultBinPath(info.ID)
+	}
+
 	stat, err := os.Stat(binPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -127,8 +148,9 @@ func (store FileStore) AsConcatableUpload(upload handler.Upload) handler.Concata
 	return upload.(*fileUpload)
 }
 
-// binPath returns the path to the file storing the binary data.
-func (store FileStore) binPath(id string) string {
+// defaultBinPath returns the path to the file storing the binary data, if it is
+// not customized using the pre-create hook.
+func (store FileStore) defaultBinPath(id string) string {
 	return filepath.Join(store.Path, id)
 }
 

pkg/filestore/filestore_test.go

@@ -245,3 +245,71 @@ func TestDeclareLength(t *testing.T) {
 	a.EqualValues(100, updatedInfo.Size)
 	a.Equal(false, updatedInfo.SizeIsDeferred)
 }
+
+// TestCustomPath tests whether the upload's destination can be customized.
+func TestCustomPath(t *testing.T) {
+	a := assert.New(t)
+
+	tmp, err := os.MkdirTemp("", "tusd-filestore-")
+	a.NoError(err)
+
+	store := FileStore{tmp}
+	ctx := context.Background()
+
+	// Create new upload
+	upload, err := store.NewUpload(ctx, handler.FileInfo{
+		ID:   "folder1/info",
+		Size: 42,
+		Storage: map[string]string{
+			"Path": "./folder2/bin",
+		},
+	})
+	a.NoError(err)
+	a.NotEqual(nil, upload)
+
+	// Check info without writing
+	info, err := upload.GetInfo(ctx)
+	a.NoError(err)
+	a.EqualValues(42, info.Size)
+	a.EqualValues(0, info.Offset)
+	a.Equal(2, len(info.Storage))
+	a.Equal("filestore", info.Storage["Type"])
+	a.Equal(filepath.Join(tmp, "./folder2/bin"), info.Storage["Path"])
+
+	// Write data to upload
+	bytesWritten, err := upload.WriteChunk(ctx, 0, strings.NewReader("hello world"))
+	a.NoError(err)
+	a.EqualValues(len("hello world"), bytesWritten)
+
+	// Check new offset
+	info, err = upload.GetInfo(ctx)
+	a.NoError(err)
+	a.EqualValues(42, info.Size)
+	a.EqualValues(11, info.Offset)
+
+	// Read content
+	reader, err := upload.GetReader(ctx)
+	a.NoError(err)
+
+	content, err := io.ReadAll(reader)
+	a.NoError(err)
+	a.Equal("hello world", string(content))
+	reader.(io.Closer).Close()
+
+	// Check that the output file and info file exist on disk
+	statInfo, err := os.Stat(filepath.Join(tmp, "folder2/bin"))
+	a.NoError(err)
+	a.True(statInfo.Mode().IsRegular())
+	a.EqualValues(11, statInfo.Size())
+	statInfo, err = os.Stat(filepath.Join(tmp, "folder1/info.info"))
+	a.NoError(err)
+	a.True(statInfo.Mode().IsRegular())
+
+	// Terminate upload
+	a.NoError(store.AsTerminatableUpload(upload).Terminate(ctx))
+
+	// Test if upload is deleted
+	upload, err = store.GetUpload(ctx, info.ID)
+	a.Equal(nil, upload)
+	a.Equal(handler.ErrNotFound, err)
+}