-
Notifications
You must be signed in to change notification settings - Fork 5.1k
Ppaul/failover ledger endpoint #51947
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 7 commits
2e30dd0
515190d
2fa1b35
af7213a
cb79689
bf8d04e
19870c2
409c510
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,288 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Text.Json; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Azure.Core; | ||
| using Azure.Core.Pipeline; | ||
|
|
||
| namespace Azure.Security.ConfidentialLedger | ||
| { | ||
| internal class ConfidentialLedgerFailoverService | ||
| { | ||
| private readonly HttpPipeline _pipeline; | ||
| private readonly ClientDiagnostics _clientDiagnostics; | ||
|
|
||
| private static ResponseClassifier _responseClassifier200; | ||
| private static ResponseClassifier ResponseClassifier200 => _responseClassifier200 ??= new StatusCodeClassifier(stackalloc ushort[] { 200 }); | ||
|
|
||
| public ConfidentialLedgerFailoverService(HttpPipeline pipeline, ClientDiagnostics clientDiagnostics) | ||
| { | ||
| _pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline)); | ||
| _clientDiagnostics = clientDiagnostics ?? throw new ArgumentNullException(nameof(clientDiagnostics)); | ||
| } | ||
| // Overloads for failover-only execution with collectionId gating. | ||
| public Task<T> ExecuteOnFailoversAsync<T>( | ||
| Uri primaryEndpoint, | ||
| Func<Uri, Task<T>> operationAsync, | ||
| string operationName, | ||
| string collectionIdGate, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| if (string.IsNullOrEmpty(collectionIdGate)) | ||
| { | ||
| return operationAsync(primaryEndpoint); // collection gating: no failover | ||
| } | ||
| return ExecuteOnFailoversAsync(primaryEndpoint, operationAsync, operationName, cancellationToken); | ||
| } | ||
|
|
||
| public T ExecuteOnFailovers<T>( | ||
| Uri primaryEndpoint, | ||
| Func<Uri, T> operationSync, | ||
| string operationName, | ||
| string collectionIdGate, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| if (string.IsNullOrEmpty(collectionIdGate)) | ||
| { | ||
| return operationSync(primaryEndpoint); | ||
| } | ||
| return ExecuteOnFailovers(primaryEndpoint, operationSync, operationName, cancellationToken); | ||
| } | ||
|
|
||
| private async Task<List<Uri>> GetFailoverEndpointsAsync( | ||
| Uri primaryEndpoint, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| var failoverEndpoints = new List<Uri>(); | ||
|
|
||
| try | ||
| { | ||
| string ledgerId = primaryEndpoint.Host.Substring(0, primaryEndpoint.Host.IndexOf('.')); | ||
|
|
||
| Uri failoverUrl = new Uri($"https://identity.confidential-ledger.core.azure.com/failover/{ledgerId}"); | ||
PallabPaul marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| using HttpMessage message = CreateFailoverRequest(failoverUrl); | ||
| Response response = await _pipeline.ProcessMessageAsync(message, new RequestContext()).ConfigureAwait(false); | ||
|
|
||
| if (response.Status == 200) | ||
| { | ||
| using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); | ||
| jsonDoc.RootElement.TryGetProperty("ledgerId", out _); // fire & forget | ||
| if (jsonDoc.RootElement.TryGetProperty("failoverLedgers", out JsonElement failoverArray)) | ||
| { | ||
| foreach (JsonElement failoverLedger in failoverArray.EnumerateArray()) | ||
| { | ||
| string failoverLedgerId = null; | ||
| try | ||
| { | ||
| switch (failoverLedger.ValueKind) | ||
| { | ||
| case JsonValueKind.String: | ||
| failoverLedgerId = failoverLedger.GetString(); | ||
| break; | ||
| case JsonValueKind.Object: | ||
| if (failoverLedger.TryGetProperty("name", out JsonElement nameProp) && nameProp.ValueKind == JsonValueKind.String) | ||
| { | ||
| failoverLedgerId = nameProp.GetString(); | ||
| } | ||
| else | ||
| { | ||
| foreach (JsonProperty prop in failoverLedger.EnumerateObject()) | ||
| { | ||
| if (prop.Value.ValueKind == JsonValueKind.String && string.Equals(prop.Name, "id", StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| failoverLedgerId = prop.Value.GetString(); | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| break; | ||
| } | ||
| } | ||
| catch (Exception) | ||
| { | ||
| // ignore element issues | ||
PallabPaul marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| if (!string.IsNullOrEmpty(failoverLedgerId)) | ||
| { | ||
| Uri endpoint = new UriBuilder(primaryEndpoint) { Host = $"{failoverLedgerId}.confidential-ledger.azure.com" }.Uri; | ||
PallabPaul marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| failoverEndpoints.Add(endpoint); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| catch (Exception) | ||
| { | ||
PallabPaul marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // suppress metadata retrieval exception | ||
| } | ||
|
|
||
| return failoverEndpoints; | ||
| } | ||
|
|
||
| private List<Uri> GetFailoverEndpoints( | ||
| Uri primaryEndpoint, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| var failoverEndpoints = new List<Uri>(); | ||
|
|
||
| try | ||
| { | ||
| // retrieving sync metadata | ||
| string ledgerId = primaryEndpoint.Host.Substring(0, primaryEndpoint.Host.IndexOf('.')); | ||
|
|
||
| Uri failoverUrl = new Uri($"https://identity.confidential-ledger.core.azure.com/failover/{ledgerId}"); | ||
PallabPaul marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| using HttpMessage message = CreateFailoverRequest(failoverUrl); | ||
| Response response = _pipeline.ProcessMessage(message, new RequestContext()); | ||
|
|
||
| if (response.Status == 200) | ||
| { | ||
| using JsonDocument jsonDoc = JsonDocument.Parse(response.Content); | ||
| jsonDoc.RootElement.TryGetProperty("ledgerId", out _); | ||
| if (jsonDoc.RootElement.TryGetProperty("failoverLedgers", out JsonElement failoverArray)) | ||
| { | ||
| foreach (JsonElement failoverLedger in failoverArray.EnumerateArray()) | ||
| { | ||
| string failoverLedgerId = null; | ||
| try | ||
| { | ||
| switch (failoverLedger.ValueKind) | ||
| { | ||
| case JsonValueKind.String: | ||
| failoverLedgerId = failoverLedger.GetString(); | ||
| break; | ||
| case JsonValueKind.Object: | ||
| if (failoverLedger.TryGetProperty("name", out JsonElement nameProp) && nameProp.ValueKind == JsonValueKind.String) | ||
| { | ||
| failoverLedgerId = nameProp.GetString(); | ||
| } | ||
| else | ||
| { | ||
| foreach (JsonProperty prop in failoverLedger.EnumerateObject()) | ||
| { | ||
| if (prop.Value.ValueKind == JsonValueKind.String && string.Equals(prop.Name, "id", StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| failoverLedgerId = prop.Value.GetString(); | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| break; | ||
| } | ||
| } | ||
| catch (Exception) | ||
| { | ||
| // ignore element issues | ||
| } | ||
|
|
||
| if (!string.IsNullOrEmpty(failoverLedgerId)) | ||
| { | ||
| Uri endpoint = new UriBuilder(primaryEndpoint) { Host = $"{failoverLedgerId}.confidential-ledger.azure.com" }.Uri; | ||
| failoverEndpoints.Add(endpoint); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| catch (Exception) | ||
| { | ||
| // suppress metadata retrieval exception | ||
| } | ||
|
|
||
| return failoverEndpoints; | ||
| } | ||
|
|
||
| private HttpMessage CreateFailoverRequest(Uri failoverUrl) | ||
| { | ||
| HttpMessage message = _pipeline.CreateMessage(new RequestContext(), ResponseClassifier200); | ||
| Request request = message.Request; | ||
|
|
||
| request.Method = RequestMethod.Get; | ||
|
|
||
| var uri = new RawRequestUriBuilder(); | ||
| uri.Reset(failoverUrl); | ||
| request.Uri = uri; | ||
|
|
||
| request.Headers.Add("Accept", "application/json"); | ||
|
|
||
| return message; | ||
| } | ||
|
|
||
| private static bool IsRetriableFailure(RequestFailedException ex) | ||
| { | ||
| // Include 404 and specific UnknownLedgerEntry error code. | ||
| return ex.Status == 404 || | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These errors need to be updated, I doubt that a failover endpoint has so many retriable conditions. 404 would mean that the ledger does not exist, |
||
| string.Equals(ex.ErrorCode, "UnknownLedgerEntry", StringComparison.OrdinalIgnoreCase) || | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should |
||
| ex.Status >= 500 || | ||
| ex.Status == 408 || | ||
| ex.Status == 429 || | ||
| ex.Status == 503 || | ||
| ex.Status == 504; | ||
| } | ||
|
|
||
| // Execute an operation only against discovered failover endpoints (skips primary). Used for specialized fallback flows. | ||
| public async Task<T> ExecuteOnFailoversAsync<T>( | ||
| Uri primaryEndpoint, | ||
| Func<Uri, Task<T>> operationAsync, | ||
| string operationName, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| List<Uri> endpoints = await GetFailoverEndpointsAsync(primaryEndpoint, cancellationToken).ConfigureAwait(false); | ||
| Exception last = null; | ||
| foreach (var ep in endpoints) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just a minor suggestion, probably not worth doing for this PR since this customer only has one failover: switch to
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is an interesting point. It is saying that the order of the failover ledgers is important and sequential. For instance if there are 2 failover ledgers and the primary fails it will redirect all traffic to the first failover ledger and then to third only if the primary and secondary fail. Third one would rarely be used in this case. If the order does not matter much it might make sense to forward requests to a randomly ordered list of failover ledgers to improve the distribution of the requests. So that when we get x million of requests they do not suddenly get all forwarded to the failover ledger but distribute that load across all of the failover ones. |
||
| { | ||
| // attempt endpoint | ||
| try | ||
| { | ||
| cancellationToken.ThrowIfCancellationRequested(); | ||
| return await operationAsync(ep).ConfigureAwait(false); | ||
| } | ||
| catch (RequestFailedException ex) when (IsRetriableFailure(ex)) | ||
| { | ||
| // endpoint failed, continue | ||
| last = ex; | ||
| } | ||
| catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) | ||
| { | ||
| throw; | ||
| } | ||
| } | ||
| throw last ?? new RequestFailedException("All failover endpoints failed in failovers mode"); | ||
| } | ||
|
|
||
| public T ExecuteOnFailovers<T>( | ||
| Uri primaryEndpoint, | ||
| Func<Uri, T> operationSync, | ||
| string operationName, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| List<Uri> endpoints = GetFailoverEndpoints(primaryEndpoint, cancellationToken); | ||
| Exception last = null; | ||
| foreach (var ep in endpoints) | ||
| { | ||
| // attempt endpoint | ||
| try | ||
| { | ||
| cancellationToken.ThrowIfCancellationRequested(); | ||
| return operationSync(ep); | ||
| } | ||
| catch (RequestFailedException ex) when (IsRetriableFailure(ex)) | ||
| { | ||
| // endpoint failed | ||
| last = ex; | ||
| } | ||
| catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) | ||
| { | ||
| throw; | ||
| } | ||
| } | ||
| throw last ?? new RequestFailedException("All failover endpoints failed in failovers mode"); | ||
| } | ||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.