Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Added Single Share File Upload Tests; Create before putrange
  • Loading branch information
amnguye committed Sep 29, 2023
commit b157b90486f4d30f277a61cd898137a68d70a0d6
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
#elif BlobDataMovementSDK
Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
#elif ShareDataMovementSDK
Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion;
#else
// If you see this error, you've included this shared source file from a
// client library that it doesn't know how to help you with. Either add
Expand All @@ -39,7 +41,7 @@ internal static class StorageVersionExtensions
/// Gets the latest version of the service supported by this SDK.
/// </summary>
public const ServiceVersion LatestVersion =
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK || ShareDataMovementSDK
ServiceVersion.V2023_08_03;
#else
ERROR_STORAGE_SERVICE_NOT_DEFINED;
Expand All @@ -49,7 +51,7 @@ internal static class StorageVersionExtensions
/// Gets the latest version of the service supported by this SDK.
/// </summary>
internal const ServiceVersion MaxVersion =
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK || ShareDataMovementSDK
ServiceVersion.V2023_11_03;
#else
ERROR_STORAGE_SERVICE_NOT_DEFINED;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_cd21a40eed"
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Azure.Core;
Expand Down Expand Up @@ -55,6 +56,32 @@ internal ShareFileStorageResource(
_etagDownloadLock = etagLock;
}

internal async Task CreateAsync(
bool overwrite,
long maxSize,
CancellationToken cancellationToken)
{
if (!overwrite)
{
// If overwrite is not enabled, we should check if the
// file exists first before creating because Create call will
// automatically overwrite the file if it already exists.
Response<bool> exists = await ShareFileClient.ExistsAsync(cancellationToken).ConfigureAwait(false);
if (exists.Value)
{
throw Errors.ShareFileAlreadyExists(ShareFileClient.Path);
}
}
await ShareFileClient.CreateAsync(
maxSize: maxSize,
httpHeaders: _options?.HttpHeaders,
metadata: _options?.FileMetadata,
smbProperties: _options?.SmbProperties,
filePermission: _options?.FilePermissions,
conditions: _options?.DestinationConditions,
cancellationToken: cancellationToken).ConfigureAwait(false);
}

protected override Task CompleteTransferAsync(
bool overwrite,
CancellationToken cancellationToken = default)
Expand All @@ -72,11 +99,21 @@ protected override async Task CopyBlockFromUriAsync(
CancellationToken cancellationToken = default)
{
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

if (range.Offset == 0)
{
await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
if (range.Length == 0)
{
return;
}
}

await ShareFileClient.UploadRangeFromUriAsync(
sourceUri: sourceResource.Uri,
range: range,
sourceRange: range,
options: _options.ToShareFileUploadRangeFromUriOptions(),
options: _options?.ToShareFileUploadRangeFromUriOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
}

Expand All @@ -91,21 +128,22 @@ protected override async Task CopyFromStreamAsync(
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

long position = options?.Position != default ? options.Position.Value : 0;
if ((streamLength == completeLength) && position == 0)

// Create the File beforehand if it hasn't been created
if (position == 0)
{
// Default to Upload
await ShareFileClient.UploadAsync(
stream,
_options.ToShareFileUploadOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
return;
await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
if (completeLength == 0)
{
return;
}
}

// Otherwise upload the Range
await ShareFileClient.UploadRangeAsync(
new HttpRange(position, streamLength),
stream,
_options.ToShareFileUploadRangeOptions(),
_options?.ToShareFileUploadRangeOptions(),
cancellationToken).ConfigureAwait(false);
}

Expand All @@ -121,7 +159,7 @@ await ShareFileClient.UploadRangeFromUriAsync(
sourceUri: sourceResource.Uri,
range: new HttpRange(0, completeLength),
sourceRange: new HttpRange(0, completeLength),
options: _options.ToShareFileUploadRangeFromUriOptions(),
options: _options?.ToShareFileUploadRangeFromUriOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
}

Expand Down Expand Up @@ -156,7 +194,7 @@ protected override async Task<StorageResourceReadStreamResult> ReadStreamAsync(
{
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
Response<ShareFileDownloadInfo> response = await ShareFileClient.DownloadAsync(
_options.ToShareFileDownloadOptions(new HttpRange(position, length)),
_options?.ToShareFileDownloadOptions(new HttpRange(position, length)),
cancellationToken).ConfigureAwait(false);
return response.Value.ToStorageResourceReadStreamResult();
}
Expand All @@ -171,4 +209,12 @@ protected override StorageResourceCheckpointData GetDestinationCheckpointData()
throw new NotImplementedException();
}
}

#pragma warning disable SA1402 // File may only contain a single type
internal partial class Errors
#pragma warning restore SA1402 // File may only contain a single type
{
public static InvalidOperationException ShareFileAlreadyExists(string pathName)
=> new InvalidOperationException($"Share File `{pathName}` already exists. Cannot overwrite file.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,16 @@
<!-- Required Shared Source for testing -->
<Compile Include="$(AzureStorageSharedSources)Constants.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)Errors.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)Errors.Clients.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StorageConnectionString.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StorageVersionExtensions.cs" LinkBase="Shared\Storage" />
<Compile Include="$(AzureStorageSharedSources)SharedAccessSignatureCredentials.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)UriQueryParamsCollection.cs" LinkBase="Shared" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageSharedTestSources)\**\*.cs" LinkBase="Shared\Storage" />
<Compile Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Files.Shares\tests\DisposingShare.cs" LinkBase="Shared\Storage" />
<Compile Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Files.Shares\tests\ShareClientTestFixtureAttribute.cs" LinkBase="Shared\Storage" />
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteFixture.cs"/>
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteNUnitFixture.cs"/>
<Compile Remove="$(AzureStorageSharedTestSources)\ClientSideEncryptionTestExtensions.cs" />
Expand All @@ -33,6 +37,13 @@
</None>
<None Include="$(AzureStorageSharedTestSources)\azurite_cert.pem" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageDataMovementSharedSources)DataTransferStatusInternal.cs" LinkBase="Shared\DataMovement" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
</ItemGroup>
<ItemGroup>
<Content Include="Resources\**">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.Storage.Files.Shares;
using Azure.Storage.Files.Shares.Tests;
using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
Azure.Storage.Files.Shares.ShareServiceClient,
Azure.Storage.Files.Shares.ShareClientOptions>;

namespace Azure.Storage.DataMovement.Files.Shares.Tests
{
internal static class ClientBuilderExtensions
{
public static string GetNewShareName(this SharesClientBuilder clientBuilder)
=> $"test-share-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewDirectoryName(this SharesClientBuilder clientBuilder)
=> $"test-directory-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewNonAsciiDirectoryName(this SharesClientBuilder clientBuilder)
=> $"test-dire¢t Ø®ϒ%3A-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewFileName(this SharesClientBuilder clientBuilder)
=> $"test-file-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewNonAsciiFileName(this SharesClientBuilder clientBuilder)
=> $"test-ƒ¡£€‽%3A-{clientBuilder.Recording.Random.NewGuid()}";

public static async Task<DisposingShare> GetTestShareAsync(
this SharesClientBuilder clientBuilder,
ShareServiceClient service = default,
string shareName = default,
IDictionary<string, string> metadata = default,
ShareClientOptions options = default)
{
service ??= clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigDefault, options);
metadata ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
shareName ??= clientBuilder.GetNewShareName();
ShareClient share = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(service.GetShareClient(shareName));
return await DisposingShare.CreateAsync(share, metadata);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System.Threading.Tasks;
using System.Collections.Generic;
using Azure.Core.TestFramework;
using Azure.Storage.Files.Shares;
using Azure.Storage.Files.Shares.Tests;
using Azure.Storage.Test.Shared;
using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
Azure.Storage.Files.Shares.ShareServiceClient,
Azure.Storage.Files.Shares.ShareClientOptions>;
using System.IO;

namespace Azure.Storage.DataMovement.Files.Shares.Tests
{
[ShareClientTestFixture]
public abstract class DataMovementShareTestBase : StorageTestBase<StorageTestEnvironment>
{
protected readonly ShareClientOptions.ServiceVersion _serviceVersion;

/// <summary>
/// Source of clients.
/// </summary>
protected SharesClientBuilder SharesClientBuilder { get; }

public DataMovementShareTestBase(bool async, ShareClientOptions.ServiceVersion serviceVersion, RecordedTestMode? mode = null)
: base(async, mode /* RecordedTestMode.Record /* to re-record */)
{
_serviceVersion = serviceVersion;
SharesClientBuilder = GetNewSharesClientBuilder(Tenants, _serviceVersion);
}

public string GetNewShareName() => SharesClientBuilder.GetNewShareName();
public string GetNewDirectoryName() => SharesClientBuilder.GetNewDirectoryName();
public string GetNewNonAsciiDirectoryName() => SharesClientBuilder.GetNewNonAsciiDirectoryName();
public string GetNewFileName() => SharesClientBuilder.GetNewFileName();
public string GetNewNonAsciiFileName() => SharesClientBuilder.GetNewNonAsciiFileName();

/// <summary>
/// Creates a new <see cref="ClientBuilder{TServiceClient, TServiceClientOptions}"/>
/// setup to generate <see cref="BlobServiceClient"/>s.
/// </summary>
/// <param name="tenants"><see cref="TenantConfigurationBuilder"/> powering this client builder.</param>
/// <param name="serviceVersion">Service version for clients to target.</param>
public SharesClientBuilder GetNewSharesClientBuilder(TenantConfigurationBuilder tenants, ShareClientOptions.ServiceVersion serviceVersion)
=> new SharesClientBuilder(
ServiceEndpoint.File,
tenants,
(uri, clientOptions) => new ShareServiceClient(uri, clientOptions),
(uri, sharedKeyCredential, clientOptions) => new ShareServiceClient(uri, sharedKeyCredential, clientOptions),
(uri, tokenCredential, clientOptions) => new ShareServiceClient(uri, tokenCredential, clientOptions),
(uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions),
() => new ShareClientOptions(serviceVersion));

public async Task<DisposingShare> GetTestShareAsync(
ShareServiceClient service = default,
string containerName = default,
IDictionary<string, string> metadata = default,
ShareClientOptions options = default)
=> await SharesClientBuilder.GetTestShareAsync(service, containerName, metadata, options);

internal async Task<ShareFileClient> CreateShareFile(
ShareDirectoryClient directoryClient,
string localSourceFile,
string fileName,
long size)
{
ShareFileClient fileClient = directoryClient.GetFileClient(fileName);

// create a new file and copy contents of stream into it, and then close the FileStream
// so the StagedUploadAsync call is not prevented from reading using its FileStream.
using Stream originalStream = await CreateLimitedMemoryStream(size);
using (FileStream fileStream = File.Create(localSourceFile))
{
// Copy source to a file, so we can verify the source against downloaded blob later
await originalStream.CopyToAsync(fileStream);
// Upload blob to storage account
originalStream.Position = 0;
await fileClient.CreateAsync(size);
await fileClient.UploadAsync(originalStream);
}
return fileClient;
}
}
}
Loading