From b50b7023dfa66fcfde2b47e5e48c6d6b9135053a Mon Sep 17 00:00:00 2001 From: Andres Paz Date: Fri, 3 Dec 2021 01:14:20 -0800 Subject: [PATCH 1/3] Adding ability to provide content type and encoding for Azure blobs --- .../Microsoft.Azure.Quantum.Client.csproj | 4 +- .../Storage/IJobStorageHelper.cs | 16 ++++++++ .../Storage/IStorageHelper.cs | 18 +++++++++ .../Storage/JobStorageHelper.cs | 26 ++++++++----- .../Storage/JobStorageHelperBase.cs | 25 +++++++----- .../Storage/LinkedStorageJobHelper.cs | 38 +++++++++---------- .../Storage/StorageHelper.cs | 26 ++++++++----- .../Utility/Compression.cs | 27 +++++++++++++ 8 files changed, 130 insertions(+), 50 deletions(-) create mode 100644 src/Azure/Azure.Quantum.Client/Utility/Compression.cs diff --git a/src/Azure/Azure.Quantum.Client/Microsoft.Azure.Quantum.Client.csproj b/src/Azure/Azure.Quantum.Client/Microsoft.Azure.Quantum.Client.csproj index 5585bc6f1b4..03d29f4fc36 100644 --- a/src/Azure/Azure.Quantum.Client/Microsoft.Azure.Quantum.Client.csproj +++ b/src/Azure/Azure.Quantum.Client/Microsoft.Azure.Quantum.Client.csproj @@ -20,10 +20,10 @@ - + - + all diff --git a/src/Azure/Azure.Quantum.Client/Storage/IJobStorageHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/IJobStorageHelper.cs index 30e058779ed..015b7809076 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/IJobStorageHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/IJobStorageHelper.cs @@ -29,6 +29,22 @@ public interface IJobStorageHelper Stream input, CancellationToken cancellationToken = default); + /// + /// Uploads the job input. + /// + /// The job id. + /// The input. + /// The MIME type indicating the content of the payload. + /// A flag to indicate if the payload should be uploaded compressed to storage. + /// The cancellation token. + /// Container uri + Input uri. + Task<(string containerUri, string inputUri)> UploadJobInputAsync( + string jobId, + Stream input, + string contentType, + bool compress, + CancellationToken cancellationToken = default); + /// /// Uploads the job program output mapping. /// diff --git a/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs index 54b7ae589fa..4239af4ef10 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs @@ -32,14 +32,32 @@ Task DownloadBlobAsync( /// Container client. /// Name of the BLOB. /// The input. + /// The MIME type indicating the content of the payload. + /// The blob encoding. /// The cancellation token. /// async task. Task UploadBlobAsync( BlobContainerClient containerClient, string blobName, Stream input, + string contentType, + string contentEncoding, CancellationToken cancellationToken = default); + /// + /// Uploads the BLOB. + /// + /// Container client. + /// Name of the BLOB. + /// The input. + /// The cancellation token. + /// async task. + Task UploadBlobAsync( + BlobContainerClient containerClient, + string blobName, + Stream input, + CancellationToken cancellationToken = default) => this.UploadBlobAsync(containerClient, blobName, input, null, null, cancellationToken); + /// /// Gets the BLOB sas URI. /// diff --git a/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelper.cs index 63234851d42..046cff470ca 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelper.cs @@ -37,28 +37,34 @@ public JobStorageHelper(string connectionString) } } - /// - /// Uploads the job input. - /// - /// The job id. - /// The input. - /// The cancellation token. - /// - /// Container uri + Input uri. - /// + /// public override async Task<(string containerUri, string inputUri)> UploadJobInputAsync( string jobId, Stream input, + string contentType, + bool compress, CancellationToken cancellationToken = default) { string containerName = GetContainerName(jobId); + string encoding = null; + Stream data = input; + + if (compress) + { + var compressedInput = new MemoryStream(); + await Compression.Compress(input, compressedInput); + data = compressedInput; + encoding = "gzip"; + } BlobContainerClient containerClient = await this.GetContainerClient(containerName); await this.StorageHelper.UploadBlobAsync( containerClient, Constants.Storage.InputBlobName, - input, + input: data, + contentType: contentType, + contentEncoding: encoding, cancellationToken); string containerUri = this.StorageHelper.GetBlobContainerSasUri( diff --git a/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelperBase.cs b/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelperBase.cs index 92ea7fdb47f..b0c0fcfd6e3 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelperBase.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/JobStorageHelperBase.cs @@ -7,7 +7,9 @@ namespace Microsoft.Azure.Quantum.Storage using System.IO; using System.Threading; using System.Threading.Tasks; + using global::Azure.Storage.Blobs; + using Microsoft.Azure.Quantum.Utility; public abstract class JobStorageHelperBase : IJobStorageHelper @@ -54,17 +56,20 @@ await this.StorageHelper.DownloadBlobAsync( return; } - /// - /// Uploads the job input. - /// - /// The job id. - /// The input. - /// The cancellation token. - /// Container uri + Input uri. + /// public abstract Task<(string containerUri, string inputUri)> UploadJobInputAsync( - string jobId, - Stream input, - CancellationToken cancellationToken = default); + string jobId, + Stream input, + string contentType, + bool compress, + CancellationToken cancellationToken = default); + + /// + public Task<(string containerUri, string inputUri)> UploadJobInputAsync( + string jobId, + Stream input, + CancellationToken cancellationToken = default) => + this.UploadJobInputAsync(jobId, input, null, false, cancellationToken); /// /// Uploads the job program output mapping. diff --git a/src/Azure/Azure.Quantum.Client/Storage/LinkedStorageJobHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/LinkedStorageJobHelper.cs index e3e59f5f0b1..b9b103a5ad8 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/LinkedStorageJobHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/LinkedStorageJobHelper.cs @@ -23,28 +23,34 @@ public LinkedStorageJobHelper(IWorkspace workspace) this.workspace = workspace; } - /// - /// Uploads the job input. - /// - /// The job id. - /// The input. - /// The cancellation token. - /// - /// Container uri + Input uri without SAS. - /// + /// public override async Task<(string containerUri, string inputUri)> UploadJobInputAsync( string jobId, Stream input, + string contentType, + bool compress, CancellationToken cancellationToken = default) { string containerName = GetContainerName(jobId); + string encoding = null; + Stream data = input; BlobContainerClient containerClient = await this.GetContainerClient(containerName); + if (compress) + { + var compressedInput = new MemoryStream(); + await Compression.Compress(input, compressedInput); + data = compressedInput; + encoding = "gzip"; + } + await this.StorageHelper.UploadBlobAsync( - containerClient, - Constants.Storage.InputBlobName, - input, + containerClient: containerClient, + blobName: Constants.Storage.InputBlobName, + input: data, + contentType: contentType, + contentEncoding: encoding, cancellationToken); Uri inputUri = containerClient @@ -54,13 +60,7 @@ await this.StorageHelper.UploadBlobAsync( return (GetUriPath(containerClient.Uri), GetUriPath(inputUri)); } - /// - /// Uploads the job program output mapping. - /// - /// The job id. - /// The job program output mapping. - /// The cancellation token. - /// Container uri + Mapping uri without SAS. + /// public override async Task<(string containerUri, string mappingUri)> UploadJobMappingAsync( string jobId, Stream mapping, diff --git a/src/Azure/Azure.Quantum.Client/Storage/StorageHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/StorageHelper.cs index b385af601e7..b896cddb582 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/StorageHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/StorageHelper.cs @@ -5,6 +5,8 @@ namespace Microsoft.Azure.Quantum.Storage { using System; using System.IO; + using System.Net.Mime; + using System.Text; using System.Threading; using System.Threading.Tasks; using global::Azure.Storage.Blobs; @@ -42,18 +44,13 @@ public async Task DownloadBlobAsync( return; } - /// - /// Uploads the BLOB. - /// - /// Container client. - /// Name of the BLOB. - /// The input. - /// The cancellation token. - /// Async task. + /// public async Task UploadBlobAsync( BlobContainerClient containerClient, string blobName, Stream input, + string contentType, + string contentEncoding, CancellationToken cancellationToken = default) { try @@ -61,9 +58,20 @@ public async Task UploadBlobAsync( // Ensure container is created await containerClient.CreateIfNotExistsAsync(PublicAccessType.None, cancellationToken: cancellationToken); + var headers = new BlobHttpHeaders + { + ContentEncoding = contentEncoding, + ContentType = contentType, + }; + + var options = new BlobUploadOptions + { + HttpHeaders = headers, + }; + // Upload blob BlobClient blob = containerClient.GetBlobClient(blobName); - await blob.UploadAsync(input, overwrite: true, cancellationToken); + await blob.UploadAsync(input, options, cancellationToken); } catch (Exception ex) { diff --git a/src/Azure/Azure.Quantum.Client/Utility/Compression.cs b/src/Azure/Azure.Quantum.Client/Utility/Compression.cs new file mode 100644 index 00000000000..647b06d4bf3 --- /dev/null +++ b/src/Azure/Azure.Quantum.Client/Utility/Compression.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Azure.Quantum.Utility +{ + using System.IO; + using System.IO.Compression; + using System.Threading.Tasks; + + internal class Compression + { + public static async Task Compress(Stream data, Stream compressedData) + { + using (var auxStream = new MemoryStream()) + using (var zipStream = new GZipStream(auxStream, CompressionMode.Compress)) + { + await data.CopyToAsync(zipStream); + await zipStream.FlushAsync(); + auxStream.Position = 0; + await auxStream.CopyToAsync(compressedData); + await compressedData.FlushAsync(); + } + + compressedData.Position = 0; + } + } +} From de0027dee05ede4adc2174573573969306e0e9f0 Mon Sep 17 00:00:00 2001 From: Andres Paz Date: Sat, 4 Dec 2021 15:57:58 -0800 Subject: [PATCH 2/3] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: César Zaragoza Cortés --- src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs index 4239af4ef10..2dbea105178 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs @@ -40,8 +40,8 @@ Task UploadBlobAsync( BlobContainerClient containerClient, string blobName, Stream input, - string contentType, - string contentEncoding, + string? contentType, + string? contentEncoding, CancellationToken cancellationToken = default); /// From 40d11414ecb98cecead9d128ebc1d9e4ca97580a Mon Sep 17 00:00:00 2001 From: Andres Paz Date: Sat, 4 Dec 2021 16:02:54 -0800 Subject: [PATCH 3/3] add #nullable enabled --- src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs b/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs index 2dbea105178..4bae7c272bd 100644 --- a/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs +++ b/src/Azure/Azure.Quantum.Client/Storage/IStorageHelper.cs @@ -1,6 +1,8 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#nullable enable + namespace Microsoft.Azure.Quantum.Storage { using System;