Add Track1 perf tests - upload

eng/Packages.Data.props

@@ -313,6 +313,7 @@
     <PackageReference Update="Microsoft.Azure.ResourceManager" Version="[1.1.0-preview]" />
     <PackageReference Update="Microsoft.Azure.Services.AppAuthentication" Version="[1.0.3, 2.0.0)" />
     <PackageReference Update="Microsoft.Azure.Storage.Blob" Version="11.1.7" />
+    <PackageReference Update="Microsoft.Azure.Storage.DataMovement" Version="2.0.5" />
     <PackageReference Update="Microsoft.Azure.Storage.File" Version="11.2.2" />
     <PackageReference Update="Microsoft.Azure.Storage.Queue" Version="11.1.7" />
     <PackageReference Update="Microsoft.Azure.Test.HttpRecorder" Version="[1.13.3, 2.0.0)" />

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/DirectoryTransferTest.cs

@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Azure.Test.Perf;
+using Microsoft.Azure.Storage.Auth;
+using Microsoft.Azure.Storage.Blob;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions
+    {
+        protected CloudBlobClient BlobClient;
+        private TimeSpan _transferTimeout;
+
+        protected static DirectoryTransferContext DefaultTransferContext => new()
+        {
+            ShouldOverwriteCallbackAsync = (_, _) => { return Task.FromResult(true); }
+        };
+
+        public DirectoryTransferTest(TOptions options) : base(options)
+        {
+            StorageCredentials credentials = new(
+                PerfTestEnvironment.Instance.StorageAccountName,
+                PerfTestEnvironment.Instance.StorageAccountKey);
+
+            CloudStorageAccount account = new(credentials, PerfTestEnvironment.Instance.StorageEndpointSuffix, useHttps: true);
+            BlobClient = account.CreateCloudBlobClient();
+            _transferTimeout = TimeSpan.FromSeconds(5 + (Options.Count * Options.Size) / (1 * 1024 * 1024));
+
+            if (Options.ChunkSize.HasValue)
+            {
+                TransferManager.Configurations.BlockSize = (int)Options.ChunkSize;
+            }
+        }
+
+        protected string CreateLocalDirectory(bool populate = false)
+        {
+            string directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+            Directory.CreateDirectory(directory);
+
+            if (populate)
+            {
+                foreach (int i in Enumerable.Range(0, Options.Count))
+                {
+                    string filePath = Path.Combine(directory, $"file{i}");
+                    using (Stream stream = RandomStream.Create(Options.Size))
+                    using (FileStream file = System.IO.File.Open(filePath, FileMode.Create))
+                    {
+                        stream.CopyTo(file);
+                    }
+                }
+            }
+
+            return directory;
+        }
+
+        protected async Task<CloudBlobContainer> CreateBlobContainerAsync(bool populate = false)
+        {
+            string containerName = $"test-{Guid.NewGuid()}".ToLowerInvariant();
+            CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
+            await container.CreateIfNotExistsAsync();
+
+            if (populate)
+            {
+                foreach (int i in Enumerable.Range(0, Options.Count))
+                {
+                    CloudBlockBlob blob = container.GetBlockBlobReference($"blob{i}");
+                    using (Stream stream = RandomStream.Create(Options.Size))
+                    {
+                        await blob.UploadFromStreamAsync(stream);
+                    }
+                }
+            }
+
+            return container;
+        }
+
+        protected void AssertTransferStatus(TransferStatus status)
+        {
+            if (status.NumberOfFilesSkipped > 0)
+            {
+                throw new Exception($"Transfer contained {status.NumberOfFilesSkipped} skipped files.");
+            }
+            if (status.NumberOfFilesFailed > 0)
+            {
+                throw new Exception($"Transfer contaiend {status.NumberOfFilesFailed} failed files.");
+            }
+        }
+    }
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/PerfTestEnvironment.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Core.TestFramework;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    /// <summary>
+    /// Represents the ambient environment in which the test suite is being run, offering access to information such as environment variables.
+    /// </summary>
+    internal sealed class PerfTestEnvironment : TestEnvironment
+    {
+        /// <summary>
+        /// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs.
+        /// </summary>
+        public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment();
+
+        /// <summary>
+        /// The storage account endpoint suffix to use for testing.
+        /// </summary>
+        public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net";
+
+        /// <summary>
+        /// The name of the Blob storage account to test against.
+        /// </summary>
+        /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
+        public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
+
+        /// <summary>
+        /// The shared access key of the Blob storage account to test against.
+        /// </summary>
+        /// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
+        public string StorageAccountKey => GetVariable("AZURE_STORAGE_ACCOUNT_KEY");
+    }
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Microsoft.Azure.Storage.DataMovement.Perf.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.Storage.DataMovement" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
+    <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
+  </ItemGroup>
+</Project>

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Options/DirectoryTransferOptions.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Test.Perf;
+using CommandLine;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    public class DirectoryTransferOptions : PerfOptions
+    {
+        [Option('c', "count", Default = 10, HelpText = "Number of items in each transfer.")]
+        public int Count { get; set; }
+
+        [Option('s', "size", Default = 1024, HelpText = "Size of each file (in bytes)")]
+        public long Size { get; set; }
+
+        [Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")]
+        public long? ChunkSize { get; set; }
+
+        // Override warmup to set default to 0
+        [Option('w', "warmup", Default = 0, HelpText = "Duration of warmup in seconds")]
+        public new int Warmup { get; set; }
+    }
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Program.cs

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Reflection;
+using Azure.Test.Perf;
+
+await PerfProgram.Main(Assembly.GetEntryAssembly(), args);

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Scenarios/UploadDirectory.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.Storage.Blob;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    public class UploadDirectory : DirectoryTransferTest<DirectoryTransferOptions>
+    {
+        private string _sourceDirectory;
+        private CloudBlobContainer _destinationContainer;
+
+        public UploadDirectory(DirectoryTransferOptions options) : base(options)
+        {
+        }
+
+        public override async Task GlobalSetupAsync()
+        {
+            await base.GlobalSetupAsync();
+            _sourceDirectory = CreateLocalDirectory(populate: true);
+            _destinationContainer = await CreateBlobContainerAsync();
+        }
+
+        public override async Task GlobalCleanupAsync()
+        {
+            await _destinationContainer.DeleteIfExistsAsync();
+            await base.GlobalCleanupAsync();
+        }
+
+        public override void Run(CancellationToken cancellationToken)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override async Task RunAsync(CancellationToken cancellationToken)
+        {
+            CloudBlobDirectory destination = _destinationContainer.GetDirectoryReference(string.Empty);
+            TransferStatus status = await TransferManager.UploadDirectoryAsync(
+                _sourceDirectory,
+                destination,
+                options: null,
+                DefaultTransferContext,
+                cancellationToken);
+            AssertTransferStatus(status);
+        }
+    }
+}

sdk/storage/Azure.Storage.DataMovement.sln

@@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.DataMovement.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{06787AFE-CA94-46EE-8F6F-0FD0C057022B}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Storage.DataMovement.Perf", "Azure.Storage.DataMovement.Blobs\perf\Microsoft.Azure.Storage.DataMovement.Perf\Microsoft.Azure.Storage.DataMovement.Perf.csproj", "{AD9565FD-86A1-41A6-8B73-E7C2C66BF891}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -93,6 +95,10 @@ Global
 		{06787AFE-CA94-46EE-8F6F-0FD0C057022B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{06787AFE-CA94-46EE-8F6F-0FD0C057022B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{06787AFE-CA94-46EE-8F6F-0FD0C057022B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AD9565FD-86A1-41A6-8B73-E7C2C66BF891}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD9565FD-86A1-41A6-8B73-E7C2C66BF891}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AD9565FD-86A1-41A6-8B73-E7C2C66BF891}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AD9565FD-86A1-41A6-8B73-E7C2C66BF891}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE