Skip to content

Ppaul/failover ledger endpoint #51947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PallabPaul wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Ppaul/failover ledger endpoint #51947

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2e30dd0
Update PostLedgerEntryOperation retry logic
PallabPaul Aug 3, 2023
515190d
merge master
Aug 7, 2025
2fa1b35
add base failover endpoint implementation
Aug 12, 2025
af7213a
add base failover implementation
Aug 12, 2025
cb79689
clean
Aug 12, 2025
bf8d04e
address collection id cases
Aug 13, 2025
19870c2
cleaned up failover service and client logic
Aug 14, 2025
409c510
address copilot comments
Aug 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
address collection id cases
  • Loading branch information
pallabpaul committed Aug 13, 2025
commit bf8d04e96b22c7482d03d718afe7dbd5f2e48b2f
204 changes: 200 additions & 4 deletions ...identialledger/Azure.Security.ConfidentialLedger/src/ConfidentialLedgerFailoverService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,37 @@ public T ExecuteWithFailover<T>(
throw lastException ?? new RequestFailedException("All endpoints failed");
}

// Overloads with collectionId gating: if collectionId is null/empty, skip failover entirely.
public Task<T> ExecuteWithFailoverAsync<T>(
Uri primaryEndpoint,
Func<Uri, Task<T>> operationAsync,
string operationName,
string collectionIdGate,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrEmpty(collectionIdGate))
{
Console.WriteLine($"[Failover] collectionId not provided; skipping failover for {operationName} and using primary endpoint {primaryEndpoint}");
return operationAsync(primaryEndpoint);
}
return ExecuteWithFailoverAsync(primaryEndpoint, operationAsync, operationName, cancellationToken);
}

public T ExecuteWithFailover<T>(
Uri primaryEndpoint,
Func<Uri, T> operationSync,
string operationName,
string collectionIdGate,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrEmpty(collectionIdGate))
{
Console.WriteLine($"[Failover] collectionId not provided; skipping failover for {operationName} and using primary endpoint {primaryEndpoint}");
return operationSync(primaryEndpoint);
}
return ExecuteWithFailover(primaryEndpoint, operationSync, operationName, cancellationToken);
}

private async Task<List<Uri>> GetFailoverEndpointsAsync(
Uri primaryEndpoint,
CancellationToken cancellationToken = default)
Expand All @@ -156,7 +187,7 @@ private async Task<List<Uri>> GetFailoverEndpointsAsync(

Uri failoverUrl = new UriBuilder(primaryEndpoint)
{
Host = "localhost", // update when failover endpoint logic is merged in
Host = "localhost",
Path = $"/failover/{ledgerId}"
}.Uri;

Expand All @@ -167,21 +198,73 @@ private async Task<List<Uri>> GetFailoverEndpointsAsync(
{
Console.WriteLine("[Failover] Metadata request succeeded.");
using JsonDocument jsonDoc = JsonDocument.Parse(response.Content);
if (jsonDoc.RootElement.TryGetProperty("ledgerId", out JsonElement primaryLedgerIdElem) && primaryLedgerIdElem.ValueKind == JsonValueKind.String)
{
Console.WriteLine($"[Failover] Metadata primary ledgerId: {primaryLedgerIdElem.GetString()}");
}

if (jsonDoc.RootElement.TryGetProperty("failoverLedgers", out JsonElement failoverArray))
{
int count = 0;
foreach (JsonElement failoverLedger in failoverArray.EnumerateArray())
{
string failoverLedgerId = failoverLedger.GetString();
string failoverLedgerId = null;
string access = null;
try
{
switch (failoverLedger.ValueKind)
{
case JsonValueKind.String:
failoverLedgerId = failoverLedger.GetString();
break;
case JsonValueKind.Object:
// Expected shape: { "name": "ledgerName", ... }
if (failoverLedger.TryGetProperty("name", out JsonElement nameProp) && nameProp.ValueKind == JsonValueKind.String)
{
failoverLedgerId = nameProp.GetString();
}
else
{
// Fall back: look for any string property that could represent the id
foreach (JsonProperty prop in failoverLedger.EnumerateObject())
{
if (prop.Value.ValueKind == JsonValueKind.String &&
string.Equals(prop.Name, "id", StringComparison.OrdinalIgnoreCase))
{
failoverLedgerId = prop.Value.GetString();
break;
}
}
}
if (failoverLedger.TryGetProperty("access", out JsonElement accessProp) && accessProp.ValueKind == JsonValueKind.String)
{
access = accessProp.GetString();
}
break;
default:
Console.WriteLine($"[Failover] Unexpected element kind {failoverLedger.ValueKind} in failoverLedgers array; skipping.");
break;
}
}
catch (Exception exElem)
{
Console.WriteLine($"[Failover] Suppressed exception parsing failover ledger element: {exElem.Message}");
}

if (!string.IsNullOrEmpty(failoverLedgerId))
{
Uri endpoint = new UriBuilder(primaryEndpoint)
{
Host = $"{failoverLedgerId}.confidential-ledger.azure.com"
}.Uri;
failoverEndpoints.Add(endpoint);
Console.WriteLine($"[Failover] Added failover endpoint {endpoint} (access={access ?? "unknown"})");
count++;
}
else
{
Console.WriteLine("[Failover] Could not extract ledger id from element; skipping.");
}
}
Console.WriteLine($"[Failover] Parsed {count} failover ledger id(s).");
}
Expand Down Expand Up @@ -227,21 +310,71 @@ private List<Uri> GetFailoverEndpoints(
{
Console.WriteLine("[Failover] Metadata request succeeded.");
using JsonDocument jsonDoc = JsonDocument.Parse(response.Content);
if (jsonDoc.RootElement.TryGetProperty("ledgerId", out JsonElement primaryLedgerIdElem) && primaryLedgerIdElem.ValueKind == JsonValueKind.String)
{
Console.WriteLine($"[Failover] Metadata primary ledgerId: {primaryLedgerIdElem.GetString()}");
}

if (jsonDoc.RootElement.TryGetProperty("failoverLedgers", out JsonElement failoverArray))
{
int count = 0;
foreach (JsonElement failoverLedger in failoverArray.EnumerateArray())
{
string failoverLedgerId = failoverLedger.GetString();
string failoverLedgerId = null;
string access = null;
try
{
switch (failoverLedger.ValueKind)
{
case JsonValueKind.String:
failoverLedgerId = failoverLedger.GetString();
break;
case JsonValueKind.Object:
if (failoverLedger.TryGetProperty("name", out JsonElement nameProp) && nameProp.ValueKind == JsonValueKind.String)
{
failoverLedgerId = nameProp.GetString();
}
else
{
foreach (JsonProperty prop in failoverLedger.EnumerateObject())
{
if (prop.Value.ValueKind == JsonValueKind.String &&
string.Equals(prop.Name, "id", StringComparison.OrdinalIgnoreCase))
{
failoverLedgerId = prop.Value.GetString();
break;
}
}
}
if (failoverLedger.TryGetProperty("access", out JsonElement accessProp) && accessProp.ValueKind == JsonValueKind.String)
{
access = accessProp.GetString();
}
break;
default:
Console.WriteLine($"[Failover] Unexpected element kind {failoverLedger.ValueKind} in failoverLedgers array; skipping.");
break;
}
}
catch (Exception exElem)
{
Console.WriteLine($"[Failover] Suppressed exception parsing failover ledger element: {exElem.Message}");
}

if (!string.IsNullOrEmpty(failoverLedgerId))
{
Uri endpoint = new UriBuilder(primaryEndpoint)
{
Host = $"{failoverLedgerId}.confidentialledger.azure.com"
Host = $"{failoverLedgerId}.confidential-ledger.azure.com"
}.Uri;
failoverEndpoints.Add(endpoint);
Console.WriteLine($"[Failover] Added failover endpoint {endpoint} (access={access ?? "unknown"})");
count++;
}
else
{
Console.WriteLine("[Failover] Could not extract ledger id from element; skipping.");
}
}
Console.WriteLine($"[Failover] Parsed {count} failover ledger id(s).");
}
Expand Down Expand Up @@ -290,5 +423,68 @@ private static bool IsRetriableFailure(RequestFailedException ex)
ex.Status == 503 ||
ex.Status == 504;
}

// Execute an operation only against discovered failover endpoints (skips primary). Used for specialized fallback flows.
public async Task<T> ExecuteOnFailoversOnlyAsync<T>(
Uri primaryEndpoint,
Func<Uri, Task<T>> operationAsync,
string operationName,
CancellationToken cancellationToken = default)
{
Console.WriteLine($"[Failover] Discovering failover endpoints for {operationName} (failovers-only mode)...");
List<Uri> endpoints = await GetFailoverEndpointsAsync(primaryEndpoint, cancellationToken).ConfigureAwait(false);
Console.WriteLine($"[Failover] Found {endpoints.Count} failover endpoint(s) for {operationName}.");
Exception last = null;
foreach (var ep in endpoints)
Copy link
Member

@taicchoumsft taicchoumsft Aug 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just a minor suggestion, probably not worth doing for this PR since this customer only has one failover: switch to Task.WhenAny or similar for the async failovers method. That should also simplify handling of the requestfailed exceptions.

Copy link
Member

@ivarprudnikov ivarprudnikov Oct 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an interesting point. It is saying that the order of the failover ledgers is important and sequential. For instance if there are 2 failover ledgers and the primary fails it will redirect all traffic to the first failover ledger and then to third only if the primary and secondary fail. Third one would rarely be used in this case.

If the order does not matter much it might make sense to forward requests to a randomly ordered list of failover ledgers to improve the distribution of the requests. So that when we get x million of requests they do not suddenly get all forwarded to the failover ledger but distribute that load across all of the failover ones.

{
Console.WriteLine($"[Failover] (FailoversOnly) Attempting {operationName} on {ep}");
try
{
cancellationToken.ThrowIfCancellationRequested();
return await operationAsync(ep).ConfigureAwait(false);
}
catch (RequestFailedException ex) when (IsRetriableFailure(ex))
{
Console.WriteLine($"[Failover] (FailoversOnly) Endpoint {ep} failed (Status {ex.Status}, Code {ex.ErrorCode}). Trying next.");
last = ex;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
}
throw last ?? new RequestFailedException("All failover endpoints failed in failovers-only mode");
}

public T ExecuteOnFailoversOnly<T>(
Uri primaryEndpoint,
Func<Uri, T> operationSync,
string operationName,
CancellationToken cancellationToken = default)
{
Console.WriteLine($"[Failover] Discovering failover endpoints for {operationName} (failovers-only mode)...");
List<Uri> endpoints = GetFailoverEndpoints(primaryEndpoint, cancellationToken);
Console.WriteLine($"[Failover] Found {endpoints.Count} failover endpoint(s) for {operationName}.");
Exception last = null;
foreach (var ep in endpoints)
{
Console.WriteLine($"[Failover] (FailoversOnly) Attempting {operationName} on {ep}");
try
{
cancellationToken.ThrowIfCancellationRequested();
return operationSync(ep);
}
catch (RequestFailedException ex) when (IsRetriableFailure(ex))
{
Console.WriteLine($"[Failover] (FailoversOnly) Endpoint {ep} failed (Status {ex.Status}, Code {ex.ErrorCode}). Trying next.");
last = ex;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
}
throw last ?? new RequestFailedException("All failover endpoints failed in failovers-only mode");
}
}
}
Loading
Loading