using System.Runtime.CompilerServices;
using System.Runtime.ExceptionServices;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MS.Microservice.AI.Abstractions;
namespace MS.Microservice.AI.Core;
/// Applies admission policies and records provider outcomes without treating reporting as provider work.
public sealed class AIProductionPipeline(
IAIRateLimiter rateLimiter,
IAICircuitBreaker circuitBreaker,
IAICostReporter costReporter,
IOptionsMonitor costOptions,
TimeProvider timeProvider,
ILogger? logger = null)
{
private readonly ILogger log = logger ?? NullLogger.Instance;
public async ValueTask ExecuteAsync(AIRequestContext context,
Func> operation, Func getUsage,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(operation);
ArgumentNullException.ThrowIfNull(getUsage);
cancellationToken.ThrowIfCancellationRequested();
await circuitBreaker.EnsureAllowedAsync(context, cancellationToken).ConfigureAwait(false);
await using var lease = await rateLimiter.AcquireAsync(context, cancellationToken).ConfigureAwait(false);
var startedAt = timeProvider.GetTimestamp();
TResponse response;
AIUsage usage;
try
{
response = await operation(cancellationToken).ConfigureAwait(false);
usage = getUsage(response);
}
catch (Exception exception)
{
await ObserveAsync(context, AIUsage.Zero, startedAt, false, exception,
exception is OperationCanceledException && cancellationToken.IsCancellationRequested, cancellationToken).ConfigureAwait(false);
throw;
}
await ObserveAsync(context, usage, startedAt, true, null, false, cancellationToken).ConfigureAwait(false);
return response;
}
public async IAsyncEnumerable ExecuteStreamAsync(AIRequestContext context,
Func> operation, Func getUsage,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(operation);
ArgumentNullException.ThrowIfNull(getUsage);
cancellationToken.ThrowIfCancellationRequested();
await circuitBreaker.EnsureAllowedAsync(context, cancellationToken).ConfigureAwait(false);
await using var lease = await rateLimiter.AcquireAsync(context, cancellationToken).ConfigureAwait(false);
var startedAt = timeProvider.GetTimestamp();
var usage = AIUsage.Zero;
var completed = false;
Exception? failure = null;
IAsyncEnumerator? enumerator = null;
try
{
try { enumerator = operation(cancellationToken).GetAsyncEnumerator(cancellationToken); }
catch (Exception exception) { failure = exception; throw; }
while (true)
{
TChunk chunk;
try
{
if (!await enumerator.MoveNextAsync().ConfigureAwait(false)) break;
chunk = enumerator.Current;
usage = getUsage(chunk) ?? usage;
}
catch (Exception exception) { failure = exception; throw; }
yield return chunk;
}
completed = true;
}
finally
{
Exception? disposalFailure = null;
if (enumerator is not null)
{
try { await enumerator.DisposeAsync().ConfigureAwait(false); }
catch (Exception exception)
{
if (failure is null) failure = disposalFailure = exception;
else LogPolicyFailure("stream_disposal", exception);
}
}
var abandoned = !completed && failure is null;
if (abandoned) failure = new OperationCanceledException("The consumer ended the stream.");
await ObserveAsync(context, usage, startedAt, completed && failure is null, failure,
abandoned || failure is OperationCanceledException && cancellationToken.IsCancellationRequested,
cancellationToken).ConfigureAwait(false);
if (disposalFailure is not null) ExceptionDispatchInfo.Capture(disposalFailure).Throw();
}
}
private async ValueTask ObserveAsync(AIRequestContext context, AIUsage usage, long startedAt,
bool succeeded, Exception? exception, bool interrupted, CancellationToken cancellationToken)
{
try
{
if (succeeded) await circuitBreaker.RecordSuccessAsync(context, cancellationToken).ConfigureAwait(false);
else if (!interrupted) await circuitBreaker.RecordFailureAsync(context, exception!, cancellationToken).ConfigureAwait(false);
}
catch (Exception policyFailure) { LogPolicyFailure("circuit_recording", policyFailure); }
if (!costOptions.CurrentValue.Enabled) return;
try
{
await costReporter.ReportAsync(new AICostRecord
{
Provider = context.Provider, Model = context.Model, Capability = context.Capability,
Scenario = context.Scenario, RequestId = context.RequestId,
InputTokens = usage.InputTokens, OutputTokens = usage.OutputTokens, TotalTokens = usage.TotalTokens,
Duration = timeProvider.GetElapsedTime(startedAt), Succeeded = succeeded,
ExceptionCategory = exception is AIException aiException ? aiException.ErrorCode : exception?.GetType().Name
}, cancellationToken).ConfigureAwait(false);
}
catch (Exception reportingFailure) { LogPolicyFailure("cost_reporting", reportingFailure); }
}
private void LogPolicyFailure(string policy, Exception exception)
=> log.LogWarning("AI {Policy} failed with {FailureType}", policy, exception.GetType().Name);
}