diff --git a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json new file mode 100644 index 000000000000..3f782fc50a20 --- /dev/null +++ b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "minor", + "changeLogMessages": [ + "Added progress tracking events to simple upload" + ] + } + ] +} diff --git a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json new file mode 100644 index 000000000000..5d67e3a8b858 --- /dev/null +++ b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Added CompleteMultipartUploadResponse to TransferUtilityUploadResponse mapping" + ] + } + ] +} diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs index d130aee20bff..7e8505ecbf69 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs @@ -99,6 +99,67 @@ internal static TransferUtilityUploadResponse MapPutObjectResponse(PutObjectResp return response; } + + /// + /// Maps a CompleteMultipartUploadResponse to TransferUtilityUploadResponse. + /// Uses the field mappings defined in mapping.json "Conversion" -> "CompleteMultipartResponse" -> "UploadResponse". + /// + /// The CompleteMultipartUploadResponse to map from + /// A new TransferUtilityUploadResponse with mapped fields + internal static TransferUtilityUploadResponse MapCompleteMultipartUploadResponse(CompleteMultipartUploadResponse source) + { + if (source == null) + return null; + + var response = new TransferUtilityUploadResponse(); + + // Map all fields as defined in mapping.json "Conversion" -> "CompleteMultipartResponse" -> "UploadResponse" + if (source.IsSetBucketKeyEnabled()) + response.BucketKeyEnabled = source.BucketKeyEnabled.GetValueOrDefault(); + + if (source.IsSetChecksumCRC32()) + response.ChecksumCRC32 = source.ChecksumCRC32; + + if (source.IsSetChecksumCRC32C()) + response.ChecksumCRC32C = source.ChecksumCRC32C; + + if (source.IsSetChecksumCRC64NVME()) + response.ChecksumCRC64NVME = source.ChecksumCRC64NVME; + + if (source.IsSetChecksumSHA1()) + response.ChecksumSHA1 = source.ChecksumSHA1; + + if (source.IsSetChecksumSHA256()) + response.ChecksumSHA256 = source.ChecksumSHA256; + + if (source.ChecksumType != null) + response.ChecksumType = source.ChecksumType; + + if (source.IsSetETag()) + response.ETag = source.ETag; + + if (source.Expiration != null) + response.Expiration = source.Expiration; + + if (source.IsSetRequestCharged()) + response.RequestCharged = source.RequestCharged; + + if (source.ServerSideEncryptionMethod != null) + response.ServerSideEncryptionMethod = source.ServerSideEncryptionMethod; + + if (source.IsSetServerSideEncryptionKeyManagementServiceKeyId()) + response.ServerSideEncryptionKeyManagementServiceKeyId = source.ServerSideEncryptionKeyManagementServiceKeyId; + + if (source.IsSetVersionId()) + response.VersionId = source.VersionId; + + // Copy response metadata + response.ResponseMetadata = source.ResponseMetadata; + response.ContentLength = source.ContentLength; + response.HttpStatusCode = source.HttpStatusCode; + + return response; + } } } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs index 95a15611d2f5..072d0ff3a6d1 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs @@ -41,12 +41,18 @@ internal partial class SimpleUploadCommand : BaseCommand IAmazonS3 _s3Client; TransferUtilityConfig _config; TransferUtilityUploadRequest _fileTransporterRequest; + long _totalTransferredBytes; + private readonly long _contentLength; internal SimpleUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config, TransferUtilityUploadRequest fileTransporterRequest) { this._s3Client = s3Client; this._config = config; this._fileTransporterRequest = fileTransporterRequest; + + // Cache content length immediately while stream is accessible to avoid ObjectDisposedException in failure scenarios + this._contentLength = this._fileTransporterRequest.ContentLength; + var fileName = fileTransporterRequest.FilePath; } @@ -103,9 +109,48 @@ private PutObjectRequest ConstructRequest() private void PutObjectProgressEventCallback(object sender, UploadProgressArgs e) { - var progressArgs = new UploadProgressArgs(e.IncrementTransferred, e.TransferredBytes, e.TotalBytes, - e.CompensationForRetry, _fileTransporterRequest.FilePath); + // Keep track of the total transferred bytes so that we can also return this value in case of failure + long transferredBytes = Interlocked.Add(ref _totalTransferredBytes, e.IncrementTransferred - e.CompensationForRetry); + + var progressArgs = new UploadProgressArgs(e.IncrementTransferred, transferredBytes, _contentLength, + e.CompensationForRetry, _fileTransporterRequest.FilePath, _fileTransporterRequest); this._fileTransporterRequest.OnRaiseProgressEvent(progressArgs); } + + private void FireTransferInitiatedEvent() + { + var initiatedArgs = new UploadInitiatedEventArgs( + request: _fileTransporterRequest, + filePath: _fileTransporterRequest.FilePath, + totalBytes: _contentLength + ); + + _fileTransporterRequest.OnRaiseTransferInitiatedEvent(initiatedArgs); + } + + private void FireTransferCompletedEvent(TransferUtilityUploadResponse response) + { + var completedArgs = new UploadCompletedEventArgs( + request: _fileTransporterRequest, + response: response, + filePath: _fileTransporterRequest.FilePath, + transferredBytes: Interlocked.Read(ref _totalTransferredBytes), + totalBytes: _contentLength + ); + + _fileTransporterRequest.OnRaiseTransferCompletedEvent(completedArgs); + } + + private void FireTransferFailedEvent() + { + var failedArgs = new UploadFailedEventArgs( + request: _fileTransporterRequest, + filePath: _fileTransporterRequest.FilePath, + transferredBytes: Interlocked.Read(ref _totalTransferredBytes), + totalBytes: _contentLength + ); + + _fileTransporterRequest.OnRaiseTransferFailedEvent(failedArgs); + } } } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs index e4c94d65044f..51680eaaba09 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs @@ -38,9 +38,20 @@ await this.AsyncThrottler.WaitAsync(cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); } + FireTransferInitiatedEvent(); + var putRequest = ConstructRequest(); - await _s3Client.PutObjectAsync(putRequest, cancellationToken) + var response = await _s3Client.PutObjectAsync(putRequest, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); + + var mappedResponse = ResponseMapper.MapPutObjectResponse(response); + + FireTransferCompletedEvent(mappedResponse); + } + catch (Exception) + { + FireTransferFailedEvent(); + throw; } finally { diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs index 868fcf697dd8..ff753b6efb13 100644 --- a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs @@ -25,6 +25,7 @@ using System.IO; using System.Text; +using Amazon.Runtime; using Amazon.Runtime.Internal; using Amazon.S3.Model; using Amazon.Util; @@ -411,6 +412,132 @@ public List TagSet /// public event EventHandler UploadProgressEvent; + /// + /// The event for UploadInitiatedEvent notifications. All + /// subscribers will be notified when a transfer operation + /// starts. + /// + /// The UploadInitiatedEvent is fired exactly once when + /// a transfer operation begins. The delegates attached to the event + /// will be passed information about the upload request and + /// total file size, but no progress information. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// UploadInitiatedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void uploadStarted(object sender, UploadInitiatedEventArgs args) + /// { + /// Console.WriteLine($"Upload started: {args.FilePath}"); + /// Console.WriteLine($"Total size: {args.TotalBytes} bytes"); + /// Console.WriteLine($"Bucket: {args.Request.BucketName}"); + /// Console.WriteLine($"Key: {args.Request.Key}"); + /// } + /// + /// 2. Add this method to the UploadInitiatedEvent delegate's invocation list + /// + /// TransferUtilityUploadRequest request = new TransferUtilityUploadRequest(); + /// request.UploadInitiatedEvent += uploadStarted; + /// + /// + public event EventHandler UploadInitiatedEvent; + + /// + /// The event for UploadCompletedEvent notifications. All + /// subscribers will be notified when a transfer operation + /// completes successfully. + /// + /// The UploadCompletedEvent is fired exactly once when + /// a transfer operation completes successfully. The delegates attached to the event + /// will be passed information about the completed upload including + /// the final response from S3 with ETag, VersionId, and other metadata. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// UploadCompletedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void uploadCompleted(object sender, UploadCompletedEventArgs args) + /// { + /// Console.WriteLine($"Upload completed: {args.FilePath}"); + /// Console.WriteLine($"Transferred: {args.TransferredBytes} bytes"); + /// Console.WriteLine($"ETag: {args.Response.ETag}"); + /// Console.WriteLine($"S3 Key: {args.Response.Key}"); + /// Console.WriteLine($"Version ID: {args.Response.VersionId}"); + /// } + /// + /// 2. Add this method to the UploadCompletedEvent delegate's invocation list + /// + /// TransferUtilityUploadRequest request = new TransferUtilityUploadRequest(); + /// request.UploadCompletedEvent += uploadCompleted; + /// + /// + public event EventHandler UploadCompletedEvent; + + /// + /// The event for UploadFailedEvent notifications. All + /// subscribers will be notified when a transfer operation + /// fails. + /// + /// The UploadFailedEvent is fired exactly once when + /// a transfer operation fails. The delegates attached to the event + /// will be passed information about the failed upload including + /// partial progress information, but no response data since the upload failed. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// UploadFailedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void uploadFailed(object sender, UploadFailedEventArgs args) + /// { + /// Console.WriteLine($"Upload failed: {args.FilePath}"); + /// Console.WriteLine($"Partial progress: {args.TransferredBytes} / {args.TotalBytes} bytes"); + /// var percent = (double)args.TransferredBytes / args.TotalBytes * 100; + /// Console.WriteLine($"Completion: {percent:F1}%"); + /// Console.WriteLine($"Bucket: {args.Request.BucketName}"); + /// Console.WriteLine($"Key: {args.Request.Key}"); + /// } + /// + /// 2. Add this method to the UploadFailedEvent delegate's invocation list + /// + /// TransferUtilityUploadRequest request = new TransferUtilityUploadRequest(); + /// request.UploadFailedEvent += uploadFailed; + /// + /// + public event EventHandler UploadFailedEvent; + + /// + /// Causes the UploadInitiatedEvent event to be fired. + /// + /// UploadInitiatedEventArgs args + internal void OnRaiseTransferInitiatedEvent(UploadInitiatedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(UploadInitiatedEvent, args, this); + } + + /// + /// Causes the UploadCompletedEvent event to be fired. + /// + /// UploadCompletedEventArgs args + internal void OnRaiseTransferCompletedEvent(UploadCompletedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(UploadCompletedEvent, args, this); + } + + /// + /// Causes the UploadFailedEvent event to be fired. + /// + /// UploadFailedEventArgs args + internal void OnRaiseTransferFailedEvent(UploadFailedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(UploadFailedEvent, args, this); + } + /// /// Causes the UploadProgressEvent event to be fired. @@ -835,11 +962,164 @@ internal UploadProgressArgs(long incrementTransferred, long transferred, long to this.CompensationForRetry = compensationForRetry; } + /// + /// Constructor for upload progress with request + /// + /// The how many bytes were transferred since last event. + /// The number of bytes transferred + /// The total number of bytes to be transferred + /// A compensation for any upstream aggregators if this event to correct their totalTransferred count, + /// in case the underlying request is retried. + /// The file being uploaded + /// The original TransferUtilityUploadRequest created by the user + internal UploadProgressArgs(long incrementTransferred, long transferred, long total, long compensationForRetry, string filePath, TransferUtilityUploadRequest request) + : base(incrementTransferred, transferred, total) + { + this.FilePath = filePath; + this.CompensationForRetry = compensationForRetry; + this.Request = request; + } + /// /// Gets the FilePath. /// public string FilePath { get; private set; } internal long CompensationForRetry { get; set; } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// + public TransferUtilityUploadRequest Request { get; internal set; } + } + + /// + /// Encapsulates the information needed when a transfer operation is initiated. + /// Provides access to the original request and total file size without any progress information. + /// + public class UploadInitiatedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the UploadInitiatedEventArgs class. + /// + /// The original TransferUtilityUploadRequest created by the user + /// The file being uploaded + /// The total number of bytes to be transferred + internal UploadInitiatedEventArgs(TransferUtilityUploadRequest request, string filePath, long totalBytes) + { + Request = request; + FilePath = filePath; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// Contains all the upload parameters and configuration. + /// + public TransferUtilityUploadRequest Request { get; private set; } + + /// + /// Gets the file path being uploaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the total number of bytes to be transferred. + /// + public long TotalBytes { get; private set; } + } + + /// + /// Encapsulates the information needed when a transfer operation completes successfully. + /// Provides access to the original request, final response, and completion details. + /// + public class UploadCompletedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the UploadCompletedEventArgs class. + /// + /// The original TransferUtilityUploadRequest created by the user + /// The unified response from Transfer Utility + /// The file that was uploaded + /// The total number of bytes transferred + /// The total number of bytes that were transferred + internal UploadCompletedEventArgs(TransferUtilityUploadRequest request, TransferUtilityUploadResponse response, string filePath, long transferredBytes, long totalBytes) + { + Request = request; + Response = response; + FilePath = filePath; + TransferredBytes = transferredBytes; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// Contains all the upload parameters and configuration. + /// + public TransferUtilityUploadRequest Request { get; private set; } + + /// + /// The unified response from Transfer Utility after successful upload completion. + /// Contains mapped fields from either PutObjectResponse (simple uploads) or CompleteMultipartUploadResponse (multipart uploads). + /// + public TransferUtilityUploadResponse Response { get; private set; } + + /// + /// Gets the file path that was uploaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the total number of bytes that were successfully transferred. + /// + public long TransferredBytes { get; private set; } + + /// + /// Gets the total number of bytes that were transferred (should equal TransferredBytes for successful uploads). + /// + public long TotalBytes { get; private set; } + } + + /// + /// Encapsulates the information needed when a transfer operation fails. + /// Provides access to the original request and partial progress information. + /// + public class UploadFailedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the UploadFailedEventArgs class. + /// + /// The original TransferUtilityUploadRequest created by the user + /// The file that was being uploaded + /// The number of bytes transferred before failure + /// The total number of bytes that should have been transferred + internal UploadFailedEventArgs(TransferUtilityUploadRequest request, string filePath, long transferredBytes, long totalBytes) + { + Request = request; + FilePath = filePath; + TransferredBytes = transferredBytes; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// Contains all the upload parameters and configuration. + /// + public TransferUtilityUploadRequest Request { get; private set; } + + /// + /// Gets the file path that was being uploaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the number of bytes that were transferred before the failure occurred. + /// + public long TransferredBytes { get; private set; } + + /// + /// Gets the total number of bytes that should have been transferred. + /// + public long TotalBytes { get; private set; } } } diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index abc767a0bdd8..3280bc13b241 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -30,6 +30,8 @@ public class TransferUtilityTests : TestBase private static string fullPath; private const string testContent = "This is the content body!"; private const string testFile = "PutObjectFile.txt"; + private static string testFilePath; + private const string testKey = "SimpleUploadProgressTotalBytesTestFile.txt"; [ClassInitialize()] public static void ClassInitialize(TestContext a) @@ -66,6 +68,7 @@ public static void ClassInitialize(TestContext a) fullPath = Path.GetFullPath(testFile); File.WriteAllText(fullPath, testContent); + testFilePath = fullPath; // Use the same file for the TotalBytes test } [ClassCleanup] @@ -105,6 +108,113 @@ public void SimpleUploadProgressTest() progressValidator.AssertOnCompletion(); } + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadInitiatedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\InitiatedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.IsTrue(args.TotalBytes > 0); + Assert.AreEqual(10 * MEG_SIZE, args.TotalBytes); + } + }; + UploadWithLifecycleEvents(fileName, 10 * MEG_SIZE, eventValidator, null, null); + eventValidator.AssertEventFired(); + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadCompletedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\CompletedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.IsNotNull(args.Response); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.AreEqual(args.TransferredBytes, args.TotalBytes); + Assert.AreEqual(10 * MEG_SIZE, args.TotalBytes); + Assert.IsTrue(!string.IsNullOrEmpty(args.Response.ETag)); + } + }; + UploadWithLifecycleEvents(fileName, 10 * MEG_SIZE, null, eventValidator, null); + eventValidator.AssertEventFired(); + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadFailedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\FailedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.IsTrue(args.TotalBytes > 0); + Assert.AreEqual(5 * MEG_SIZE, args.TotalBytes); + // For failed uploads, transferred bytes should be less than or equal to total bytes + Assert.IsTrue(args.TransferredBytes <= args.TotalBytes); + } + }; + + // Use invalid bucket name to force failure + var invalidBucketName = "invalid-bucket-name-" + Guid.NewGuid().ToString(); + + try + { + UploadWithLifecycleEventsAndBucket(fileName, 5 * MEG_SIZE, invalidBucketName, null, null, eventValidator); + Assert.Fail("Expected an exception to be thrown for invalid bucket"); + } + catch (AmazonS3Exception) + { + // Expected exception - the failed event should have been fired + eventValidator.AssertEventFired(); + } + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadCompleteLifecycleTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\CompleteLifecycle"); + + var initiatedValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.AreEqual(8 * MEG_SIZE, args.TotalBytes); + } + }; + + var completedValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.IsNotNull(args.Response); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.AreEqual(args.TransferredBytes, args.TotalBytes); + Assert.AreEqual(8 * MEG_SIZE, args.TotalBytes); + } + }; + + UploadWithLifecycleEvents(fileName, 8 * MEG_SIZE, initiatedValidator, completedValidator, null); + + initiatedValidator.AssertEventFired(); + completedValidator.AssertEventFired(); + } + [TestMethod] [TestCategory("S3")] public void SimpleUpload() @@ -337,41 +447,36 @@ public void UploadUnSeekableStreamFileSizeEqualToPartBufferSize() } [TestMethod] - [TestCategory("S3")] - public void UploadUnseekableStreamFileSizeBetweenMinPartSizeAndPartBufferSize() + public void SimpleUploadProgressTotalBytesTest() { - var client = Client; - var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\BetweenMinPartSizeAndPartBufferSize"); - var path = Path.Combine(BasePath, fileName); - // there was a bug where the transfer utility was uploading 13MB file - // when the file size was between 5MB and (5MB + 8192). 8192 is the s3Client.Config.BufferSize - var fileSize = 5 * MEG_SIZE + 1; - - UtilityMethods.GenerateFile(path, fileSize); - //take the generated file and turn it into an unseekable stream - - var stream = GenerateUnseekableStreamFromFile(path); - using (var tu = new Amazon.S3.Transfer.TransferUtility(client)) + var transferConfig = new TransferUtilityConfig() { - tu.Upload(stream, bucketName, fileName); + MinSizeBeforePartUpload = 20 * MEG_SIZE, + }; - var metadata = Client.GetObjectMetadata(new GetObjectMetadataRequest + var progressValidator = new TransferProgressValidator + { + Validate = (progress) => { - BucketName = bucketName, - Key = fileName - }); - Assert.AreEqual(fileSize, metadata.ContentLength); + Assert.IsTrue(progress.TotalBytes > 0, "TotalBytes should be greater than 0"); + Assert.AreEqual(testContent.Length, progress.TotalBytes, "TotalBytes should equal file length"); + } + }; - //Download the file and validate content of downloaded file is equal. - var downloadPath = path + ".download"; - var downloadRequest = new TransferUtilityDownloadRequest + using (var fileTransferUtility = new TransferUtility(Client, transferConfig)) + { + var request = new TransferUtilityUploadRequest() { BucketName = bucketName, - Key = fileName, - FilePath = downloadPath + FilePath = testFilePath, + Key = testKey }; - tu.Download(downloadRequest); - UtilityMethods.CompareFiles(path, downloadPath); + + request.UploadProgressEvent += progressValidator.OnProgressEvent; + + fileTransferUtility.Upload(request); + + progressValidator.AssertOnCompletion(); } } @@ -1374,6 +1479,87 @@ public void OnProgressEvent(object sender, T progress) } } } + + class TransferLifecycleEventValidator + { + public Action Validate { get; set; } + public bool EventFired { get; private set; } + public Exception EventException { get; private set; } + + public void OnEventFired(object sender, T eventArgs) + { + try + { + EventFired = true; + Console.WriteLine("Lifecycle Event Fired: {0}", typeof(T).Name); + Validate?.Invoke(eventArgs); + } + catch (Exception ex) + { + EventException = ex; + Console.WriteLine("Exception caught in lifecycle event: {0}", ex.Message); + throw; + } + } + + public void AssertEventFired() + { + if (EventException != null) + throw EventException; + + // Add some time for the background thread to finish before checking + for (int retries = 1; retries < 5 && !EventFired; retries++) + { + Thread.Sleep(1000 * retries); + } + Assert.IsTrue(EventFired, $"{typeof(T).Name} event was not fired"); + } + } + + void UploadWithLifecycleEvents(string fileName, long size, + TransferLifecycleEventValidator initiatedValidator, + TransferLifecycleEventValidator completedValidator, + TransferLifecycleEventValidator failedValidator) + { + UploadWithLifecycleEventsAndBucket(fileName, size, bucketName, initiatedValidator, completedValidator, failedValidator); + } + + void UploadWithLifecycleEventsAndBucket(string fileName, long size, string targetBucketName, + TransferLifecycleEventValidator initiatedValidator, + TransferLifecycleEventValidator completedValidator, + TransferLifecycleEventValidator failedValidator) + { + var key = fileName; + var path = Path.Combine(BasePath, fileName); + UtilityMethods.GenerateFile(path, size); + + var config = new TransferUtilityConfig(); + var transferUtility = new TransferUtility(Client, config); + var request = new TransferUtilityUploadRequest + { + BucketName = targetBucketName, + FilePath = path, + Key = key, + ContentType = octetStreamContentType + }; + + if (initiatedValidator != null) + { + request.UploadInitiatedEvent += initiatedValidator.OnEventFired; + } + + if (completedValidator != null) + { + request.UploadCompletedEvent += completedValidator.OnEventFired; + } + + if (failedValidator != null) + { + request.UploadFailedEvent += failedValidator.OnEventFired; + } + + transferUtility.Upload(request); + } private class UnseekableStream : MemoryStream { private readonly bool _setZeroLengthStream; diff --git a/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs b/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs index 0b57959f2b5a..01f251f4c415 100644 --- a/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs +++ b/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs @@ -254,6 +254,117 @@ public void ValidateTransferUtilityUploadResponseDefinitionCompleteness() "TransferUtilityUploadResponse"); } + [TestMethod] + [TestCategory("S3")] + public void MapCompleteMultipartUploadResponse_AllMappedProperties_WorkCorrectly() + { + // Get the expected mappings from JSON + var completeMultipartMappings = _mappingJson.RootElement + .GetProperty("Conversion") + .GetProperty("CompleteMultipartResponse") + .GetProperty("UploadResponse") + .EnumerateArray() + .Select(prop => prop.GetString()) + .ToList(); + + // Create source object with dynamically generated test data + var sourceResponse = new CompleteMultipartUploadResponse(); + var sourceType = typeof(CompleteMultipartUploadResponse); + var testDataValues = new Dictionary(); + + // Generate test data for each mapped property + foreach (var propertyName in completeMultipartMappings) + { + // Resolve alias to actual property name + var resolvedPropertyName = ResolvePropertyName(propertyName); + var sourceProperty = sourceType.GetProperty(resolvedPropertyName); + if (sourceProperty?.CanWrite == true) + { + var testValue = GenerateTestValue(sourceProperty.PropertyType, propertyName); + sourceProperty.SetValue(sourceResponse, testValue); + testDataValues[propertyName] = testValue; + } + } + + // Add inherited properties for comprehensive testing + sourceResponse.HttpStatusCode = HttpStatusCode.OK; + sourceResponse.ContentLength = 2048; + + // Map the response + var mappedResponse = ResponseMapper.MapCompleteMultipartUploadResponse(sourceResponse); + Assert.IsNotNull(mappedResponse, "Mapped response should not be null"); + + // Verify all mapped properties using reflection + var targetType = typeof(TransferUtilityUploadResponse); + var failedAssertions = new List(); + + foreach (var propertyName in completeMultipartMappings) + { + // Resolve alias to actual property name for reflection lookups + var resolvedPropertyName = ResolvePropertyName(propertyName); + var sourceProperty = sourceType.GetProperty(resolvedPropertyName); + var targetProperty = targetType.GetProperty(resolvedPropertyName); + + if (sourceProperty == null) + { + failedAssertions.Add($"Source property '{propertyName}' (resolved to: {resolvedPropertyName}) not found in CompleteMultipartUploadResponse"); + continue; + } + + if (targetProperty == null) + { + failedAssertions.Add($"Target property '{propertyName}' (resolved to: {resolvedPropertyName}) not found in TransferUtilityUploadResponse"); + continue; + } + + var sourceValue = sourceProperty.GetValue(sourceResponse); + var targetValue = targetProperty.GetValue(mappedResponse); + + // Special handling for complex object comparisons + if (!AreValuesEqual(sourceValue, targetValue)) + { + failedAssertions.Add($"{propertyName}: Expected '{sourceValue ?? "null"}', got '{targetValue ?? "null"}'"); + } + } + + // Test inherited properties + Assert.AreEqual(sourceResponse.HttpStatusCode, mappedResponse.HttpStatusCode, "HttpStatusCode should match"); + Assert.AreEqual(sourceResponse.ContentLength, mappedResponse.ContentLength, "ContentLength should match"); + + // Report any failures + if (failedAssertions.Any()) + { + Assert.Fail($"Property mapping failures:\n{string.Join("\n", failedAssertions)}"); + } + } + + [TestMethod] + [TestCategory("S3")] + public void MapCompleteMultipartUploadResponse_NullValues_HandledCorrectly() + { + // Test null handling scenarios + var testCases = new[] + { + // Test null Expiration + new CompleteMultipartUploadResponse { Expiration = null }, + + // Test null enum conversions + new CompleteMultipartUploadResponse { ChecksumType = null, RequestCharged = null, ServerSideEncryption = null } + }; + + foreach (var testCase in testCases) + { + var mapped = ResponseMapper.MapCompleteMultipartUploadResponse(testCase); + Assert.IsNotNull(mapped, "Response should always be mappable"); + + // Test null handling + if (testCase.Expiration == null) + { + Assert.IsNull(mapped.Expiration, "Null Expiration should map to null"); + } + } + } + [TestMethod] [TestCategory("S3")] public void ValidateCompleteMultipartUploadResponseConversionCompleteness()