Send azure upload (#1334)

* Add direct upload api endpoints

* Create azure upload service

* Update max file size

* Update send file upload test

* Move internationalization string to correct document

* Allow for one shot blob uploads

* Remove unused helper

* Use FileUploadService

Fallback to legacy method on old server implementations.
This commit is contained in:
Matt Gibson 2021-03-29 09:45:04 -05:00 committed by GitHub
parent ab04759b0e
commit 13ffbe911a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 400 additions and 40 deletions

View File

@ -215,7 +215,7 @@
Clicked="ChooseFile_Clicked" />
<Label
Margin="0, 5, 0, 0"
Text="{u:I18n MaxFileSize}"
Text="{u:I18n MaxFileSizeSend}"
StyleClass="text-sm, text-muted"
HorizontalOptions="FillAndExpand"
HorizontalTextAlignment="Center" />
@ -513,4 +513,4 @@
</ResourceDictionary>
</ContentPage.Resources>
</pages:BaseContentPage>
</pages:BaseContentPage>

View File

@ -327,7 +327,7 @@ namespace Bit.App.Pages
AppResources.AnErrorHasOccurred);
return false;
}
if (FileData.Length > 104857600) // 100 MB
if (FileData.Length > 524288000) // 500 MB
{
await _platformUtilsService.ShowDialogAsync(AppResources.MaxFileSize,
AppResources.AnErrorHasOccurred);

View File

@ -927,6 +927,9 @@
</data>
<data name="MaxFileSize" xml:space="preserve">
<value>Maximum file size is 100 MB.</value>
</data>
<data name="MaxFileSizeSend" xml:space="preserve">
<value>Maximum file size is 500 MB.</value>
</data>
<data name="UpdateKey" xml:space="preserve">
<value>You cannot use this feature until you update your encryption key.</value>

View File

@ -56,7 +56,11 @@ namespace Bit.Core.Abstractions
Task<SendResponse> GetSendAsync(string id);
Task<SendResponse> PostSendAsync(SendRequest request);
Task<SendFileUploadDataResponse> PostFileTypeSendAsync(SendRequest request);
Task PostSendFileAsync(string sendId, string fileId, MultipartFormDataContent data);
[Obsolete("Mar 25 2021: This method has been deprecated in favor of direct uploads. This method still exists for backward compatibility with old server versions.")]
Task<SendResponse> PostSendFileAsync(MultipartFormDataContent data);
Task<SendFileUploadDataResponse> RenewFileUploadUrlAsync(string sendId, string fileId);
Task<SendResponse> PutSendAsync(string id, SendRequest request);
Task<SendResponse> PutSendRemovePasswordAsync(string id);
Task DeleteSendAsync(string id);

View File

@ -0,0 +1,10 @@
using System;
using System.Threading.Tasks;
namespace Bit.Core.Abstractions
{
public interface IAzureFileUploadService
{
Task Upload(string uri, byte[] data, Func<Task<string>> renewalCallback);
}
}

View File

@ -0,0 +1,9 @@
using System.Threading.Tasks;
using Bit.Core.Models.Domain;
using Bit.Core.Models.Response;
namespace Bit.Core.Abstractions {
public interface IFileUploadService {
Task UploadSendFileAsync(SendFileUploadDataResponse uploadData, CipherString fileName, byte[] encryptedFileData);
}
}

View File

@ -0,0 +1,9 @@
using System;
namespace Bit.Core.Enums
{
public enum FileUploadType
{
Direct = 0,
Azure = 1,
}
}

View File

@ -0,0 +1,12 @@
using System;
using Bit.Core.Enums;
namespace Bit.Core.Models.Response
{
public class SendFileUploadDataResponse
{
public string Url { get; set; }
public FileUploadType FileUploadType { get; set; }
public SendResponse SendResponse { get; set; }
}
}

View File

@ -223,9 +223,19 @@ namespace Bit.Core.Services
public Task<SendResponse> PostSendAsync(SendRequest request) =>
SendAsync<SendRequest, SendResponse>(HttpMethod.Post, "/sends", request, true, true);
public Task<SendFileUploadDataResponse> PostFileTypeSendAsync(SendRequest request) =>
SendAsync<SendRequest, SendFileUploadDataResponse>(HttpMethod.Post, "/sends/file/v2", request, true, true);
public Task PostSendFileAsync(string sendId, string fileId, MultipartFormDataContent data) =>
SendAsync<MultipartFormDataContent, object>(HttpMethod.Post, $"/sends/{sendId}/file/{fileId}", data, true, false);
[Obsolete("Mar 25 2021: This method has been deprecated in favor of direct uploads. This method still exists for backward compatibility with old server versions.")]
public Task<SendResponse> PostSendFileAsync(MultipartFormDataContent data) =>
SendAsync<MultipartFormDataContent, SendResponse>(HttpMethod.Post, "/sends/file", data, true, true);
public Task<SendFileUploadDataResponse> RenewFileUploadUrlAsync(string sendId, string fileId) =>
SendAsync<object, SendFileUploadDataResponse>(HttpMethod.Get, $"/sends/{sendId}/file/{fileId}", null, true, true);
public Task<SendResponse> PutSendAsync(string id, SendRequest request) =>
SendAsync<SendRequest, SendResponse>(HttpMethod.Put, $"/sends/{id}", request, true, true);

View File

@ -0,0 +1,196 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using Bit.Core.Abstractions;
using Bit.Core.Utilities;
namespace Bit.Core.Services
{
public class AzureFileUploadService : IAzureFileUploadService
{
private const long MAX_SINGLE_BLOB_UPLOAD_SIZE = 256 * 1024 * 1024; // 256 MiB
private const int MAX_BLOCKS_PER_BLOB = 50000;
private const decimal MAX_MOBILE_BLOCK_SIZE = 5 * 1024 * 1024; // 5 MB
private readonly HttpClient _httpClient = new HttpClient();
public AzureFileUploadService()
{
_httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue()
{
NoCache = true,
};
}
public async Task Upload(string uri, byte[] data, Func<Task<string>> renewalCallback)
{
if (data.Length <= MAX_SINGLE_BLOB_UPLOAD_SIZE)
{
await AzureUploadBlob(uri, data);
}
else
{
await AzureUploadBlocks(uri, data, renewalCallback);
}
}
private async Task AzureUploadBlob(string uri, byte[] data)
{
using (var requestMessage = new HttpRequestMessage())
{
var uriBuilder = new UriBuilder(uri);
var paramValues = HttpUtility.ParseQueryString(uriBuilder.Query);
requestMessage.Headers.Add("x-ms-date", DateTime.UtcNow.ToString("R"));
requestMessage.Headers.Add("x-ms-version", paramValues["sv"]);
requestMessage.Headers.Add("x-ms-blob-type", "BlockBlob");
requestMessage.Content = new ByteArrayContent(data);
requestMessage.Version = new Version(1, 0);
requestMessage.Method = HttpMethod.Put;
requestMessage.RequestUri = uriBuilder.Uri;
var blobResponse = await _httpClient.SendAsync(requestMessage);
if (blobResponse.StatusCode != HttpStatusCode.Created)
{
throw new Exception("Failed to create Azure blob");
}
}
}
private async Task AzureUploadBlocks(string uri, byte[] data, Func<Task<string>> renewalFunc)
{
_httpClient.Timeout = TimeSpan.FromHours(3);
var baseParams = HttpUtility.ParseQueryString(CoreHelpers.GetUri(uri).Query);
var blockSize = MaxBlockSize(baseParams["sv"]);
var blockIndex = 0;
var numBlocks = Math.Ceiling((decimal)data.Length / blockSize);
var blocksStaged = new List<string>();
if (numBlocks > MAX_BLOCKS_PER_BLOB)
{
throw new Exception($"Cannot upload file, exceeds maximum size of {blockSize * MAX_BLOCKS_PER_BLOB}");
}
while (blockIndex < numBlocks)
{
uri = await RenewUriIfNecessary(uri, renewalFunc);
var blockUriBuilder = new UriBuilder(uri);
var blockId = EncodeBlockId(blockIndex);
var blockParams = HttpUtility.ParseQueryString(blockUriBuilder.Query);
blockParams.Add("comp", "block");
blockParams.Add("blockid", blockId);
blockUriBuilder.Query = blockParams.ToString();
using (var requestMessage = new HttpRequestMessage())
{
requestMessage.Headers.Add("x-ms-date", DateTime.UtcNow.ToString("R"));
requestMessage.Headers.Add("x-ms-version", baseParams["sv"]);
requestMessage.Headers.Add("x-ms-blob-type", "BlockBlob");
requestMessage.Content = new ByteArrayContent(data.Skip(blockIndex * blockSize).Take(blockSize).ToArray());
requestMessage.Version = new Version(1, 0);
requestMessage.Method = HttpMethod.Put;
requestMessage.RequestUri = blockUriBuilder.Uri;
var blockResponse = await _httpClient.SendAsync(requestMessage);
if (blockResponse.StatusCode != HttpStatusCode.Created)
{
throw new Exception("Failed to create Azure block");
}
}
blocksStaged.Add(blockId);
blockIndex++;
}
using (var requestMessage = new HttpRequestMessage())
{
uri = await RenewUriIfNecessary(uri, renewalFunc);
var blockListXml = GenerateBlockListXml(blocksStaged);
var blockListUriBuilder = new UriBuilder(uri);
var blockListParams = HttpUtility.ParseQueryString(blockListUriBuilder.Query);
blockListParams.Add("comp", "blocklist");
blockListUriBuilder.Query = blockListParams.ToString();
requestMessage.Headers.Add("x-ms-date", DateTime.UtcNow.ToString("R"));
requestMessage.Headers.Add("x-ms-version", baseParams["sv"]);
requestMessage.Content = new StringContent(blockListXml);
requestMessage.Version = new Version(1, 0);
requestMessage.Method = HttpMethod.Put;
requestMessage.RequestUri = blockListUriBuilder.Uri;
var blockListResponse = await _httpClient.SendAsync(requestMessage);
if (blockListResponse.StatusCode != HttpStatusCode.Created)
{
throw new Exception("Failed to PUT Azure block list");
}
}
}
private async Task<string> RenewUriIfNecessary(string uri, Func<Task<string>> renewalFunc)
{
var uriParams = HttpUtility.ParseQueryString(CoreHelpers.GetUri(uri).Query);
if (DateTime.TryParse(uriParams.Get("se") ?? "", out DateTime expiry) && expiry < DateTime.UtcNow.AddSeconds(1))
{
return await renewalFunc();
}
return uri;
}
private string GenerateBlockListXml(List<string> blocksStaged)
{
var xml = new StringBuilder("<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList>");
foreach(var blockId in blocksStaged)
{
xml.Append($"<Latest>{blockId}</Latest>");
}
xml.Append("</BlockList>");
return xml.ToString();
}
private string EncodeBlockId(int index)
{
// Encoded blockId max size is 64, so pre-encoding max size is 48
var paddedString = index.ToString("D48");
return Convert.ToBase64String(Encoding.UTF8.GetBytes(paddedString));
}
private int MaxBlockSize(string version)
{
long maxSize = 4194304L; // 4 MiB
if (CompareAzureVersions(version, "2019-12-12") >= 0)
{
maxSize = 4194304000L; // 4000 MiB
}
else if (CompareAzureVersions(version, "2016-05-31") >= 0)
{
maxSize = 104857600L; // 100 MiB
}
return maxSize > MAX_MOBILE_BLOCK_SIZE ? (int)MAX_MOBILE_BLOCK_SIZE : (int) maxSize;
}
private int CompareAzureVersions(string a, string b)
{
var v1Parts = a.Split('-').Select(p => int.Parse(p));
var v2Parts = b.Split('-').Select(p => int.Parse(p));
return a[0] != b[0] ? a[0] - b[0] :
a[1] != b[1] ? a[1] - b[1] :
a[2] != b[2] ? a[2] - b[2] :
0;
}
}
}

View File

@ -0,0 +1,26 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
namespace Bit.Core.Services
{
public class BitwardenFileUploadService
{
public BitwardenFileUploadService(ApiService apiService)
{
_apiService = apiService;
}
private readonly ApiService _apiService;
public async Task Upload(string encryptedFileName, byte[] encryptedFileData, Func<MultipartFormDataContent, Task> apiCall)
{
var fd = new MultipartFormDataContent($"--BWMobileFormBoundary{DateTime.UtcNow.Ticks}")
{
{ new ByteArrayContent(encryptedFileData), "data", encryptedFileName }
};
await apiCall(fd);
}
}
}

View File

@ -0,0 +1,51 @@
using System.Threading.Tasks;
using Bit.Core.Abstractions;
using Bit.Core.Models.Domain;
using Bit.Core.Models.Response;
using Bit.Core.Enums;
using System;
namespace Bit.Core.Services {
public class FileUploadService : IFileUploadService
{
public FileUploadService(ApiService apiService)
{
_apiService = apiService;
_bitwardenFileUploadService = new BitwardenFileUploadService(apiService);
_azureFileUploadService = new AzureFileUploadService();
}
private readonly BitwardenFileUploadService _bitwardenFileUploadService;
private readonly AzureFileUploadService _azureFileUploadService;
private readonly ApiService _apiService;
public async Task UploadSendFileAsync(SendFileUploadDataResponse uploadData, CipherString fileName, byte[] encryptedFileData)
{
try
{
switch (uploadData.FileUploadType)
{
case FileUploadType.Direct:
await _bitwardenFileUploadService.Upload(fileName.EncryptedString, encryptedFileData,
fd => _apiService.PostSendFileAsync(uploadData.SendResponse.Id, uploadData.SendResponse.File.Id, fd));
break;
case FileUploadType.Azure:
Func<Task<string>> renewalCallback = async () =>
{
var response = await _apiService.RenewFileUploadUrlAsync(uploadData.SendResponse.Id, uploadData.SendResponse.File.Id);
return response.Url;
};
await _azureFileUploadService.Upload(uploadData.Url, encryptedFileData, renewalCallback);
break;
default:
throw new Exception("Unknown file upload type");
}
}
catch (Exception e)
{
await _apiService.DeleteSendAsync(uploadData.SendResponse.Id);
throw e;
}
}
}
}

View File

@ -1,11 +1,12 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Bit.Core.Abstractions;
using Bit.Core.Enums;
using Bit.Core.Exceptions;
using Bit.Core.Models.Data;
using Bit.Core.Models.Domain;
using Bit.Core.Models.Request;
@ -25,11 +26,13 @@ namespace Bit.Core.Services
private readonly II18nService _i18nService;
private readonly ICryptoFunctionService _cryptoFunctionService;
private Task<List<SendView>> _getAllDecryptedTask;
private readonly IFileUploadService _fileUploadService;
public SendService(
ICryptoService cryptoService,
IUserService userService,
IApiService apiService,
IFileUploadService fileUploadService,
IStorageService storageService,
II18nService i18nService,
ICryptoFunctionService cryptoFunctionService)
@ -37,6 +40,7 @@ namespace Bit.Core.Services
_cryptoService = cryptoService;
_userService = userService;
_apiService = apiService;
_fileUploadService = fileUploadService;
_storageService = storageService;
_i18nService = i18nService;
_cryptoFunctionService = cryptoFunctionService;
@ -195,7 +199,7 @@ namespace Bit.Core.Services
public async Task<string> SaveWithServerAsync(Send send, byte[] encryptedFileData)
{
var request = new SendRequest(send, encryptedFileData?.LongLength);
SendResponse response;
SendResponse response = default;
if (send.Id == null)
{
switch (send.Type)
@ -204,13 +208,23 @@ namespace Bit.Core.Services
response = await _apiService.PostSendAsync(request);
break;
case SendType.File:
var fd = new MultipartFormDataContent($"--BWMobileFormBoundary{DateTime.UtcNow.Ticks}")
{
{ new StringContent(JsonConvert.SerializeObject(request)), "model" },
{ new ByteArrayContent(encryptedFileData), "data", send.File.FileName.EncryptedString }
};
try{
var uploadDataResponse = await _apiService.PostFileTypeSendAsync(request);
response = uploadDataResponse.SendResponse;
response = await _apiService.PostSendFileAsync(fd);
await _fileUploadService.UploadSendFileAsync(uploadDataResponse, send.File.FileName, encryptedFileData);
}
catch (ApiException e) when (e.Error.StatusCode == HttpStatusCode.NotFound)
{
response = await LegacyServerSendFileUpload(request, send, encryptedFileData);
}
catch (Exception e)
{
if (response != default){
await _apiService.DeleteSendAsync(response.Id);
}
throw e;
}
break;
default:
throw new NotImplementedException($"Cannot save unknown Send type {send.Type}");
@ -227,6 +241,17 @@ namespace Bit.Core.Services
return response.Id;
}
[Obsolete("Mar 25 2021: This method has been deprecated in favor of direct uploads. This method still exists for backward compatibility with old server versions.")]
private async Task<SendResponse> LegacyServerSendFileUpload(SendRequest request, Send send, byte[] encryptedFileData) {
var fd = new MultipartFormDataContent($"--BWMobileFormBoundary{DateTime.UtcNow.Ticks}")
{
{ new StringContent(JsonConvert.SerializeObject(request)), "model" },
{ new ByteArrayContent(encryptedFileData), "data", send.File.FileName.EncryptedString }
};
return await _apiService.PostSendFileAsync(fd);
}
public async Task UpsertAsync(params SendData[] sends)
{
var userId = await _userService.GetUserIdAsync();

View File

@ -80,7 +80,7 @@ namespace Bit.Core.Utilities
return null;
}
private static Uri GetUri(string uriString)
public static Uri GetUri(string uriString)
{
if (string.IsNullOrWhiteSpace(uriString))
{

View File

@ -40,13 +40,14 @@ namespace Bit.Core.Utilities
var appIdService = new AppIdService(storageService);
var userService = new UserService(storageService, tokenService);
var settingsService = new SettingsService(userService, storageService);
var fileUploadService = new FileUploadService(apiService);
var cipherService = new CipherService(cryptoService, userService, settingsService, apiService,
storageService, i18nService, () => searchService, clearCipherCacheKey, allClearCipherCacheKeys);
var folderService = new FolderService(cryptoService, userService, apiService, storageService,
i18nService, cipherService);
var collectionService = new CollectionService(cryptoService, userService, storageService, i18nService);
var sendService = new SendService(cryptoService, userService, apiService, storageService, i18nService,
cryptoFunctionService);
var sendService = new SendService(cryptoService, userService, apiService, fileUploadService, storageService,
i18nService, cryptoFunctionService);
searchService = new SearchService(cipherService, sendService);
var vaultTimeoutService = new VaultTimeoutService(cryptoService, userService, platformUtilsService,
storageService, folderService, cipherService, collectionService, searchService, messagingService, tokenService,

View File

@ -21,6 +21,8 @@ using Bit.Core.Models.Request;
using Bit.Core.Test.AutoFixture;
using System.Linq.Expressions;
using Bit.Core.Models.View;
using Bit.Core.Exceptions;
using NSubstitute.ExceptionExtensions;
namespace Bit.Core.Test.Services
{
@ -172,7 +174,6 @@ namespace Bit.Core.Test.Services
send.Id = null;
sutProvider.GetDependency<IUserService>().GetUserIdAsync().Returns(userId);
sutProvider.GetDependency<IApiService>().PostSendAsync(Arg.Any<SendRequest>()).Returns(response);
sutProvider.GetDependency<IApiService>().PostSendFileAsync(Arg.Any<MultipartFormDataContent>()).Returns(response);
var fileContentBytes = Encoding.UTF8.GetBytes("This is the file content");
@ -200,40 +201,43 @@ namespace Bit.Core.Test.Services
[Theory]
[InlineCustomAutoData(new[] { typeof(SutProviderCustomization), typeof(FileSendCustomization) })]
public async Task SaveWithServerAsync_NewFileSend_Success(SutProvider<SendService> sutProvider, string userId, SendResponse response, Send send)
public async Task SaveWithServerAsync_NewFileSend_AzureUpload_Success(SutProvider<SendService> sutProvider, string userId, SendFileUploadDataResponse response, Send send)
{
send.Id = null;
response.FileUploadType = FileUploadType.Azure;
sutProvider.GetDependency<IUserService>().GetUserIdAsync().Returns(userId);
sutProvider.GetDependency<IApiService>().PostFileTypeSendAsync(Arg.Any<SendRequest>()).Returns(response);
var fileContentBytes = Encoding.UTF8.GetBytes("This is the file content");
await sutProvider.Sut.SaveWithServerAsync(send, fileContentBytes);
switch (send.Type)
{
case SendType.File:
await sutProvider.GetDependency<IFileUploadService>().Received(1).UploadSendFileAsync(response, send.File.FileName, fileContentBytes);
break;
case SendType.Text:
default:
throw new Exception("Untested send type");
}
}
[Theory]
[InlineCustomAutoData(new[] { typeof(SutProviderCustomization), typeof(FileSendCustomization) })]
public async Task SaveWithServerAsync_NewFileSend_LegacyFallback_Success(SutProvider<SendService> sutProvider, string userId, Send send, SendResponse response)
{
send.Id = null;
sutProvider.GetDependency<IUserService>().GetUserIdAsync().Returns(userId);
sutProvider.GetDependency<IApiService>().PostSendAsync(Arg.Any<SendRequest>()).Returns(response);
var error = new ErrorResponse(null, System.Net.HttpStatusCode.NotFound);
sutProvider.GetDependency<IApiService>().PostFileTypeSendAsync(Arg.Any<SendRequest>()).Throws(new ApiException(error));
sutProvider.GetDependency<IApiService>().PostSendFileAsync(Arg.Any<MultipartFormDataContent>()).Returns(response);
var fileContentBytes = Encoding.UTF8.GetBytes("This is the file content");
await sutProvider.Sut.SaveWithServerAsync(send, fileContentBytes);
Predicate<MultipartFormDataContent> formDataPredicate = fd =>
{
Assert.Equal(2, fd.Count()); // expect a request and file content
var expectedRequest = JsonConvert.SerializeObject(new SendRequest(send, fileContentBytes?.LongLength));
var actualRequest = fd.First().ReadAsStringAsync().GetAwaiter().GetResult();
Assert.Equal(expectedRequest, actualRequest);
var actualFileContent = fd.Skip(1).First().ReadAsByteArrayAsync().GetAwaiter().GetResult();
Assert.Equal(fileContentBytes, actualFileContent);
return true;
};
switch (send.Type)
{
case SendType.File:
await sutProvider.GetDependency<IApiService>().Received(1)
.PostSendFileAsync(Arg.Is<MultipartFormDataContent>(f => formDataPredicate(f)));
break;
case SendType.Text:
default:
throw new Exception("Untested send type");
}
await sutProvider.GetDependency<IApiService>().Received(1).PostSendFileAsync(Arg.Any<MultipartFormDataContent>());
}
[Theory]