Skip to content

Commit d4e4802

Browse files
mahendra-googlemahendra-google
committed
feat(Storage): Enable full object checksum validation for resumable uploads
1 parent 6f45d2e commit d4e4802

File tree

3 files changed

+198
-81
lines changed

3 files changed

+198
-81
lines changed

‎apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs‎

Lines changed: 12 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -264,10 +264,8 @@ public void UploadObject_InvalidHash_None()
264264
var name = IdGenerator.FromGuid();
265265
var bucket = _fixture.MultiVersionBucket;
266266
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.None };
267-
// Upload succeeds despite the data being broken.
268-
client.UploadObject(bucket, name, null, stream, options);
269-
// The object should contain our "wrong" bytes.
270-
ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
267+
var exception = Assert.Throws<GoogleApiException>(() => client.UploadObject(bucket, name, null, stream, options));
268+
Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
271269
}
272270

273271
[Fact]
@@ -280,9 +278,8 @@ public void UploadObject_InvalidHash_ThrowOnly()
280278
var name = IdGenerator.FromGuid();
281279
var bucket = _fixture.MultiVersionBucket;
282280
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.ThrowOnly };
283-
Assert.Throws<UploadValidationException>(() => client.UploadObject(bucket, name, null, stream, options));
284-
// We don't delete the object, so it's still present.
285-
ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
281+
var exception = Assert.Throws<GoogleApiException>(() => client.UploadObject(bucket, name, null, stream, options));
282+
Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
286283
}
287284

288285
[Fact]
@@ -295,28 +292,12 @@ public void UploadObject_InvalidHash_DeleteAndThrow()
295292
var name = IdGenerator.FromGuid();
296293
var bucket = _fixture.MultiVersionBucket;
297294
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
298-
Assert.Throws<UploadValidationException>(() => client.UploadObject(bucket, name, null, stream, options));
295+
var exception = Assert.Throws<GoogleApiException>(() => client.UploadObject(bucket, name, null, stream, options));
296+
Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
299297
var notFound = Assert.Throws<GoogleApiException>(() => _fixture.Client.GetObject(bucket, name));
300298
Assert.Equal(HttpStatusCode.NotFound, notFound.HttpStatusCode);
301299
}
302300

303-
[Fact]
304-
public void UploadObject_InvalidHash_DeleteAndThrow_DeleteFails()
305-
{
306-
var client = StorageClient.Create();
307-
var interceptor = new BreakUploadInterceptor();
308-
client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor);
309-
client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(new BreakDeleteInterceptor());
310-
var stream = GenerateData(50);
311-
var name = IdGenerator.FromGuid();
312-
var bucket = _fixture.MultiVersionBucket;
313-
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
314-
var ex = Assert.Throws<UploadValidationException>(() => client.UploadObject(bucket, name, null, stream, options));
315-
Assert.NotNull(ex.AdditionalFailures);
316-
// The deletion failed, so the uploaded object still exists.
317-
ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
318-
}
319-
320301
[Fact]
321302
public async Task UploadObjectAsync_InvalidHash_None()
322303
{
@@ -327,10 +308,8 @@ public async Task UploadObjectAsync_InvalidHash_None()
327308
var name = IdGenerator.FromGuid();
328309
var bucket = _fixture.MultiVersionBucket;
329310
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.None };
330-
// Upload succeeds despite the data being broken.
331-
await client.UploadObjectAsync(bucket, name, null, stream, options);
332-
// The object should contain our "wrong" bytes.
333-
ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
311+
var exception = await Assert.ThrowsAsync<GoogleApiException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
312+
Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
334313
}
335314

336315
[Fact]
@@ -343,9 +322,8 @@ public async Task UploadObjectAsync_InvalidHash_ThrowOnly()
343322
var name = IdGenerator.FromGuid();
344323
var bucket = _fixture.MultiVersionBucket;
345324
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.ThrowOnly };
346-
await Assert.ThrowsAsync<UploadValidationException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
347-
// We don't delete the object, so it's still present.
348-
ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
325+
var exception = await Assert.ThrowsAsync<GoogleApiException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
326+
Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
349327
}
350328

351329
[Fact]
@@ -359,28 +337,12 @@ public async Task UploadObjectAsync_InvalidHash_DeleteAndThrow()
359337
var name = IdGenerator.FromGuid();
360338
var bucket = _fixture.MultiVersionBucket;
361339
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
362-
await Assert.ThrowsAsync<UploadValidationException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
340+
var exception = await Assert.ThrowsAsync<GoogleApiException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
341+
Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
363342
var notFound = await Assert.ThrowsAsync<GoogleApiException>(() => _fixture.Client.GetObjectAsync(bucket, name));
364343
Assert.Equal(HttpStatusCode.NotFound, notFound.HttpStatusCode);
365344
}
366345

367-
[Fact]
368-
public async Task UploadObjectAsync_InvalidHash_DeleteAndThrow_DeleteFails()
369-
{
370-
var client = StorageClient.Create();
371-
var interceptor = new BreakUploadInterceptor();
372-
client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor);
373-
client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(new BreakDeleteInterceptor());
374-
var stream = GenerateData(50);
375-
var name = IdGenerator.FromGuid();
376-
var bucket = _fixture.MultiVersionBucket;
377-
var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
378-
var ex = await Assert.ThrowsAsync<UploadValidationException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
379-
Assert.NotNull(ex.AdditionalFailures);
380-
// The deletion failed, so the uploaded object still exists.
381-
ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
382-
}
383-
384346
[Fact]
385347
public async Task InitiateUploadSessionAsync_NegativeLength()
386348
{
@@ -460,21 +422,6 @@ public async Task InterceptAsync(HttpRequestMessage request, CancellationToken c
460422
}
461423
}
462424

463-
private class BreakDeleteInterceptor : IHttpExecuteInterceptor
464-
{
465-
public Task InterceptAsync(HttpRequestMessage request, CancellationToken cancellationToken)
466-
{
467-
// We only care about Delete requests
468-
if (request.Method == HttpMethod.Delete)
469-
{
470-
// Ugly but effective hack: replace the generation URL parameter so that we add a leading 9,
471-
// so the generation we try to delete is the wrong one.
472-
request.RequestUri = new Uri(request.RequestUri.ToString().Replace("generation=", "generation=9"));
473-
}
474-
return Task.FromResult(0);
475-
}
476-
}
477-
478425
private Object GetExistingObject()
479426
{
480427
var obj = _fixture.Client.UploadObject(_fixture.MultiVersionBucket, IdGenerator.FromGuid(), "application/octet-stream", GenerateData(100));

‎apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs‎

Lines changed: 144 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2017 Google Inc. All Rights Reserved.
1+
// Copyright 2017 Google Inc. All Rights Reserved.
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -12,12 +12,16 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
using Google.Apis.Http;
1516
using Google.Apis.Services;
17+
using Google.Apis.Upload;
1618
using System;
1719
using System.IO;
20+
using System.Linq;
1821
using System.Net.Http;
22+
using System.Threading;
23+
using System.Threading.Tasks;
1924
using static Google.Apis.Storage.v1.ObjectsResource;
20-
using Google.Apis.Upload;
2125

2226
namespace Google.Cloud.Storage.V1
2327
{
@@ -26,12 +30,150 @@ namespace Google.Cloud.Storage.V1
2630
/// </summary>
2731
internal sealed class CustomMediaUpload : InsertMediaUpload
2832
{
33+
private readonly Stream _stream;
34+
private readonly Crc32cHashInterceptor _interceptor;
35+
private readonly IClientService _service;
36+
2937
public CustomMediaUpload(IClientService service, Apis.Storage.v1.Data.Object body, string bucket,
3038
Stream stream, string contentType)
3139
: base(service, body, bucket, stream, contentType)
3240
{
41+
_stream = stream;
42+
_service = service;
43+
_interceptor = new Crc32cHashInterceptor(this, _stream, _service);
44+
_service?.HttpClient?.MessageHandler?.AddExecuteInterceptor(_interceptor);
3345
}
3446

3547
internal new ResumableUploadOptions Options => base.Options;
48+
49+
private sealed class Crc32cHashInterceptor : IHttpExecuteInterceptor
50+
{
51+
private const string GoogleHashHeader = "x-goog-hash";
52+
private const int ReadBufferSize = 81920;
53+
private readonly Stream _stream;
54+
private readonly IClientService _service;
55+
private readonly CustomMediaUpload _owner;
56+
private Uri _uploadUri;
57+
58+
public Crc32cHashInterceptor(CustomMediaUpload owner, Stream stream, IClientService service)
59+
{
60+
_stream = stream;
61+
_service = service;
62+
_owner = owner;
63+
_owner.UploadSessionData += OnSessionData;
64+
_owner.ProgressChanged += OnProgressChanged;
65+
}
66+
67+
public async Task InterceptAsync(HttpRequestMessage request, CancellationToken cancellationToken)
68+
{
69+
if (_uploadUri != null && !_uploadUri.Equals(request.RequestUri))
70+
{
71+
return;
72+
}
73+
74+
if (request.Method == System.Net.Http.HttpMethod.Put && request.Content?.Headers.Contains("Content-Range") is true)
75+
{
76+
string rangeHeader = request.Content.Headers.GetValues("Content-Range").First();
77+
78+
if (IsFinalChunk(rangeHeader))
79+
{
80+
if (!_stream.CanSeek)
81+
{
82+
return;
83+
}
84+
85+
long originalPosition = _stream.Position;
86+
try
87+
{
88+
_stream.Position = 0;
89+
90+
var hasher = new Crc32c();
91+
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(ReadBufferSize);
92+
try
93+
{
94+
int bytesRead;
95+
while ((bytesRead = await _stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0)
96+
{
97+
hasher.UpdateHash(buffer, 0, bytesRead);
98+
}
99+
}
100+
finally
101+
{
102+
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
103+
}
104+
105+
byte[] hash = hasher.GetHash();
106+
string calculatedHash = Convert.ToBase64String(hash);
107+
request.Headers.TryAddWithoutValidation(GoogleHashHeader, $"crc32c={calculatedHash}");
108+
}
109+
finally
110+
{
111+
_stream.Position = originalPosition;
112+
}
113+
}
114+
}
115+
}
116+
117+
private void OnSessionData(IUploadSessionData data)
118+
{
119+
_uploadUri = data.UploadUri;
120+
_owner.UploadSessionData -= OnSessionData;
121+
}
122+
123+
private void OnProgressChanged(IUploadProgress progress)
124+
{
125+
if (progress.Status == UploadStatus.Completed || progress.Status == UploadStatus.Failed)
126+
{
127+
// Clean up when upload is finished.
128+
_service?.HttpClient?.MessageHandler?.RemoveExecuteInterceptor(this);
129+
_owner.ProgressChanged -= OnProgressChanged;
130+
}
131+
}
132+
133+
private bool IsFinalChunk(string rangeHeader)
134+
{
135+
// Expected format: "bytes {start}-{end}/{total}" or "bytes */{total}" for the final request.
136+
// We are interested in the final chunk of a known-size upload.
137+
const string prefix = "bytes ";
138+
if (!rangeHeader.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
139+
{
140+
return false;
141+
}
142+
143+
ReadOnlySpan<char> span = rangeHeader.AsSpan(prefix.Length);
144+
int slashIndex = span.IndexOf('/');
145+
if (slashIndex == -1)
146+
{
147+
return false;
148+
}
149+
150+
var totalSpan = span.Slice(slashIndex + 1);
151+
if (totalSpan.IsEmpty || totalSpan[0] == '*')
152+
{
153+
return false;
154+
}
155+
156+
if (!long.TryParse(totalSpan.ToString(), System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out long totalSize))
157+
{
158+
return false;
159+
}
160+
161+
var rangeSpan = span.Slice(0, slashIndex);
162+
int dashIndex = rangeSpan.IndexOf('-');
163+
if (dashIndex == -1)
164+
{
165+
return false;
166+
}
167+
168+
var endByteSpan = rangeSpan.Slice(dashIndex + 1);
169+
if (!long.TryParse(endByteSpan.ToString(), System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out long endByte))
170+
{
171+
return false;
172+
}
173+
174+
// If endByte is the last byte of the file, it's the final chunk.
175+
return (endByte + 1) == totalSize;
176+
}
177+
}
36178
}
37179
}

‎apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -229,29 +229,57 @@ internal async Task<Object> ExecuteAsync(CancellationToken cancellationToken)
229229
private sealed class LengthOnlyStream : Stream
230230
{
231231
private readonly long? _length;
232+
private long _position;
232233
internal LengthOnlyStream(long? length) => _length = length;
233234

234235
public override long Length => _length ?? throw new NotSupportedException();
235236
public override bool CanSeek => _length.HasValue;
236-
237-
public override bool CanRead => throw new NotImplementedException();
238-
public override bool CanWrite => throw new NotImplementedException();
237+
public override bool CanRead => true;
238+
public override bool CanWrite => false;
239239

240240
public override long Position
241241
{
242-
get => throw new NotImplementedException();
243-
set => throw new NotImplementedException();
242+
get => _position;
243+
set
244+
{
245+
if (!CanSeek) throw new NotSupportedException();
246+
if (value < 0 || value > Length) throw new ArgumentOutOfRangeException(nameof(value));
247+
_position = value;
248+
}
249+
}
250+
251+
public override int Read(byte[] buffer, int offset, int count)
252+
{
253+
if (!_length.HasValue)
254+
{
255+
return 0;
256+
}
257+
long remaining = _length.Value - _position;
258+
if (remaining <= 0) return 0;
259+
260+
int toRead = (int) Math.Min(count, remaining);
261+
262+
Array.Clear(buffer, offset, toRead);
263+
264+
_position += toRead;
265+
return toRead;
266+
}
267+
268+
public override void Flush() { }
269+
270+
public override long Seek(long offset, SeekOrigin origin)
271+
{
272+
switch (origin)
273+
{
274+
case SeekOrigin.Begin: Position = offset; break;
275+
case SeekOrigin.Current: Position += offset; break;
276+
case SeekOrigin.End: Position = Length + offset; break;
277+
}
278+
return Position;
244279
}
245280

246-
public override void Flush() => throw new NotImplementedException();
247-
public override int Read(byte[] buffer, int offset, int count) =>
248-
throw new NotImplementedException();
249-
public override long Seek(long offset, SeekOrigin origin) =>
250-
throw new NotImplementedException();
251-
public override void SetLength(long value) =>
252-
throw new NotImplementedException();
253-
public override void Write(byte[] buffer, int offset, int count) =>
254-
throw new NotImplementedException();
281+
public override void SetLength(long value) => throw new NotSupportedException();
282+
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
255283
}
256284
}
257285
}

0 commit comments

Comments
 (0)