Add stream file transfer (#1002)

Author: nginx-nickc

api/grpc/mpi/v1/files.pb.go

@@ -27,6 +27,55 @@ service FileService {
     rpc GetFile(GetFileRequest) returns (GetFileResponse) {}
     // Update a file from the Agent to the Server
     rpc UpdateFile(UpdateFileRequest) returns (UpdateFileResponse) {}
+
+    // GetFileStream requests the file content in chunks. MP and agent should agree on size to use stream
+    // vs non-stream. For smaller files, it may be more efficient to not-stream.
+    rpc GetFileStream(GetFileRequest) returns (stream FileDataChunk) {}
+
+    // UpdateFileStream uploads the file content in streams. MP and agent should agree on size to use stream
+    // vs non-stream. For smaller files, it may be more efficient to not-stream.
+    rpc UpdateFileStream(stream FileDataChunk) returns (UpdateFileResponse) {}
+}
+
+// Represents a data chunk for streaming file transfer.
+// For any Stream file transfer, following assumptions should be asserted (by implementation):
+// - invalid to contain more or less than one FileDataChunkHeaders
+// - invalid to have FileDataChunkContents before FileDataChunkHeaders
+// - invalid to have more/fewer FileDataChunkContents than FileDataChunkHeader.chunks
+// - invalid to have two FileDataChunkContents with same chunk_id
+// - invalid to have FileDataChunkContent with zero-length data
+// - invalid to have FileDataChunk message without either header or content
+// - hash of the combined contents should match FileDataChunkHeader.file_meta.hash
+// - total size of the combined contents should match FileDataChunkHeader.file_meta.size
+// - chunk_size should be less than the gRPC max message size
+message FileDataChunk {
+    // meta regarding the transfer request
+    mpi.v1.MessageMeta meta = 1;
+    oneof chunk {
+        // Chunk header
+        FileDataChunkHeader header = 2;
+        // Chunk data
+        FileDataChunkContent content = 3;
+    }
+}
+
+// Represents a chunked resource Header
+message FileDataChunkHeader {
+    // meta regarding the file, help identity the file name, size, hash, perm
+    // receiver should validate the hash against the combined contents
+    FileMeta file_meta = 1;
+    // total number of chunks expected in the transfer
+    uint32 chunks = 2 [(buf.validate.field).uint32 = { gt: 0 }];
+    // max size of individual chunks, can be undersized if EOF
+    uint32 chunk_size = 3 [(buf.validate.field).uint32 = { gt: 0 }];
+}
+
+// Represents a chunked resource chunk
+message FileDataChunkContent {
+    // chunk id, i.e. x of y, zero-indexed
+    uint32 chunk_id = 1;
+    // chunk data, should be at most chunk_size
+    bytes data = 2;
 }
 
 // Represents a request payload for a file overview

api/grpc/mpi/v1/files.pb.validate.go

@@ -8,3 +8,5 @@ package v1
 //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6@v6.8.1 -generate
 //counterfeiter:generate . CommandServiceClient
 //counterfeiter:generate . FileServiceClient
+//counterfeiter:generate . FileService_GetFileStreamServer
+//counterfeiter:generate . FileService_GetFileStreamClient