using System.Security.Cryptography; using System.Text; using System.Transactions; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; namespace MS.Microservice.Tasks.EFCore; /// Stores jobs in the host model; Stage joins its transaction and all other operations use fresh scopes. public sealed class EfCoreJobStore(TContext stagingContext, IServiceScopeFactory scopes, TimeProvider clock) : IJobStore, IJobRetentionStore where TContext : DbContext { private static readonly UTF8Encoding StrictUtf8 = new(false, true); public async Task SubmitAsync(JobSubmission submission, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(submission); try { return await IndependentlyAsync(async context => { var existing = await FindAsync(context, submission.Scope, submission.JobId, cancellationToken); if (existing is not null) return Match(existing, submission); var entry = NewEntry(submission); context.Add(entry); await context.SaveChangesAsync(cancellationToken); return entry.Snapshot(); }); } catch (DbUpdateException) { // A competing submit can win the unique key. Query only after disposing the failed context. var existing = await IndependentlyAsync(context => FindAsync(context, submission.Scope, submission.JobId, cancellationToken)); if (existing is null) throw; return Match(existing, submission); } } public async Task StageAsync(JobSubmission submission, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(submission); EnsureModel(stagingContext); if (stagingContext.Database.CurrentTransaction is null) throw new InvalidOperationException("Stage requires the caller's active DbContext transaction; use SubmitAsync for independent submission."); cancellationToken.ThrowIfCancellationRequested(); var hash = ScopeHash(submission.Scope); var existing = stagingContext.Set().Local.SingleOrDefault(entry => entry.ScopeHash == hash && entry.Id == submission.JobId) ?? await FindAsync(stagingContext, submission.Scope, submission.JobId, cancellationToken); if (existing is not null) return Match(existing, submission); var job = NewEntry(submission); stagingContext.Add(job); return job.Snapshot(); } public Task GetAsync(string scope, Guid jobId, CancellationToken cancellationToken = default) => IndependentlyAsync(async context => (await FindAsync(context, scope, jobId, cancellationToken))?.Snapshot()); public Task> ListAsync(string scope, JobState? state = null, int take = 50, CancellationToken cancellationToken = default) { var hash = ScopeHash(scope); ValidateTake(take); if (state is { } value && !Enum.IsDefined(value)) throw new ArgumentOutOfRangeException(nameof(state)); return IndependentlyAsync>(async context => { var query = Jobs(context).Where(entry => entry.ScopeHash == hash); if (state is { } filter) query = query.Where(entry => entry.State == filter); var entries = await query.OrderByDescending(entry => entry.CreatedAtUtcTicks).ThenBy(entry => entry.Id) .Take(take).ToListAsync(cancellationToken); return entries.Select(entry => entry.Snapshot()).ToArray(); }); } public Task> GetAttemptsAsync(string scope, Guid jobId, CancellationToken cancellationToken = default) { var hash = Identity(scope, jobId); return IndependentlyAsync>(async context => { var entries = await context.Set().AsNoTracking() .Where(entry => entry.ScopeHash == hash && entry.JobId == jobId) .OrderBy(entry => entry.Number).ToListAsync(cancellationToken); return entries.Select(entry => entry.Snapshot()).ToArray(); }); } public Task> GetDueAsync(string queue = "default", int take = 100, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(queue); if (queue.Length > 64) throw new ArgumentOutOfRangeException(nameof(queue)); ValidateTake(take); var now = clock.GetUtcNow().UtcTicks; return IndependentlyAsync>(async context => { var entries = await Jobs(context).Where(entry => entry.Queue == queue && entry.State == JobState.Queued && !entry.CancellationRequested && entry.AttemptCount < entry.MaximumAttempts && entry.NextRunAtUtcTicks <= now) .OrderBy(entry => entry.NextRunAtUtcTicks).ThenBy(entry => entry.CreatedAtUtcTicks).ThenBy(entry => entry.Id) .Take(take).ToListAsync(cancellationToken); return entries.Select(entry => entry.Snapshot()).ToArray(); }); } public Task TryStartAsync(string scope, Guid jobId, CancellationToken cancellationToken = default) { var hash = Identity(scope, jobId); return IndependentlyAsync(async context => { var now = clock.GetUtcNow().UtcTicks; var token = Guid.NewGuid(); await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); var changed = await Jobs(context).Where(entry => entry.ScopeHash == hash && entry.Id == jobId && entry.State == JobState.Queued && !entry.CancellationRequested && entry.AttemptCount < entry.MaximumAttempts && entry.NextRunAtUtcTicks <= now) .ExecuteUpdateAsync(setters => setters.SetProperty(entry => entry.State, JobState.Running) .SetProperty(entry => entry.AttemptCount, entry => entry.AttemptCount + 1) .SetProperty(entry => entry.AttemptToken, token) .SetProperty(entry => entry.UpdatedAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); if (changed == 0) return null; var job = await Jobs(context).SingleAsync(entry => entry.ScopeHash == hash && entry.Id == jobId, cancellationToken); context.Add(new JobAttemptEntry { ScopeHash = hash, JobId = jobId, Number = job.AttemptCount, Token = token, State = JobState.Running, StartedAtUtcTicks = now }); await context.SaveChangesAsync(cancellationToken); await transaction.CommitAsync(cancellationToken); return new(job.Snapshot(), new(scope, jobId, token)); }); } public Task> GetRunningExecutionsAsync(int take = 100, CancellationToken cancellationToken = default) { ValidateTake(take); return IndependentlyAsync>(async context => { var entries = await Jobs(context).Where(entry => entry.State == JobState.Running) .OrderBy(entry => entry.CreatedAtUtcTicks).ThenBy(entry => entry.Id).Take(take).ToListAsync(cancellationToken); return entries.Select(entry => new JobExecution(entry.Snapshot(), new(entry.Scope, entry.Id, entry.AttemptToken ?? throw new InvalidOperationException("A running job has no execution token.")))).ToArray(); }); } public Task GetExecutionStatusAsync(JobAttemptToken attempt, CancellationToken cancellationToken = default) { ValidateAttempt(attempt); return IndependentlyAsync(async context => { var requested = await Owned(context, attempt).Select(entry => (bool?)entry.CancellationRequested) .SingleOrDefaultAsync(cancellationToken); return requested is null ? JobExecutionStatus.Ended : requested.Value ? JobExecutionStatus.CancellationRequested : JobExecutionStatus.Running; }); } public Task ReportProgressAsync(JobAttemptToken attempt, int progress, string? message = null, CancellationToken cancellationToken = default) { ValidateAttempt(attempt); if (progress is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(progress)); if (message?.Length > 1024) throw new ArgumentOutOfRangeException(nameof(message)); return IndependentlyAsync(async context => { var now = clock.GetUtcNow().UtcTicks; await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); var changed = await Owned(context, attempt).Where(entry => !entry.CancellationRequested && entry.Progress <= progress) .ExecuteUpdateAsync(setters => setters.SetProperty(entry => entry.Progress, progress) .SetProperty(entry => entry.ProgressMessage, message) .SetProperty(entry => entry.UpdatedAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); if (changed == 0) return false; var updated = await CurrentAttempt(context, attempt).ExecuteUpdateAsync(setters => setters .SetProperty(entry => entry.Progress, progress).SetProperty(entry => entry.ProgressMessage, message), cancellationToken); RequireAttempt(updated); await transaction.CommitAsync(cancellationToken); return true; }); } public Task CompleteAsync(JobAttemptToken attempt, string? resultReference = null, CancellationToken cancellationToken = default) { if (resultReference is not null) { ArgumentException.ThrowIfNullOrWhiteSpace(resultReference); if (resultReference.Length > 4096) throw new ArgumentOutOfRangeException(nameof(resultReference)); } return FinishAsync(attempt, JobState.Succeeded, null, resultReference, cancellationToken); } public Task FailAsync(JobAttemptToken attempt, string errorCode, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(errorCode); if (errorCode.Length > 128) throw new ArgumentOutOfRangeException(nameof(errorCode)); return FinishAsync(attempt, JobState.Failed, errorCode, null, cancellationToken); } public Task InterruptAsync(JobAttemptToken attempt, CancellationToken cancellationToken = default) => FinishAsync(attempt, JobState.Interrupted, "interrupted", null, cancellationToken); public Task RecoverAsync(JobAttemptToken attempt, bool allowRetry, CancellationToken cancellationToken = default) => FinishAsync(attempt, JobState.Interrupted, "interrupted", null, cancellationToken, allowRetry); public Task CancelAsync(string scope, Guid jobId, CancellationToken cancellationToken = default) { var hash = Identity(scope, jobId); return IndependentlyAsync(async context => { var now = clock.GetUtcNow().UtcTicks; var query = Jobs(context).Where(entry => entry.ScopeHash == hash && entry.Id == jobId); // Separate the transitions so SET evaluation order cannot change their meaning across providers. var changed = await query.Where(entry => entry.State == JobState.Queued) .ExecuteUpdateAsync(setters => setters.SetProperty(entry => entry.CancellationRequested, true) .SetProperty(entry => entry.FinishedAtUtcTicks, now) .SetProperty(entry => entry.State, JobState.Cancelled) .SetProperty(entry => entry.UpdatedAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); if (changed == 1) return JobUpdateResult.Applied; changed = await query.Where(entry => entry.State == JobState.Running) .ExecuteUpdateAsync(setters => setters.SetProperty(entry => entry.CancellationRequested, true) .SetProperty(entry => entry.UpdatedAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); if (changed == 1) return JobUpdateResult.Applied; var existing = await query.SingleOrDefaultAsync(cancellationToken); return existing is null ? JobUpdateResult.NotFound : existing.State == JobState.Cancelled ? JobUpdateResult.Applied : JobUpdateResult.Rejected; }); } public Task RetryAsync(string scope, Guid jobId, DateTimeOffset? notBeforeUtc = null, CancellationToken cancellationToken = default) { var hash = Identity(scope, jobId); return IndependentlyAsync(async context => { var now = clock.GetUtcNow().UtcTicks; var next = notBeforeUtc?.UtcTicks ?? now; var query = Jobs(context).Where(entry => entry.ScopeHash == hash && entry.Id == jobId); var changed = await query.Where(entry => (entry.State == JobState.Failed || entry.State == JobState.Cancelled || entry.State == JobState.Interrupted) && entry.AttemptCount < entry.MaximumAttempts) .ExecuteUpdateAsync(setters => setters.SetProperty(entry => entry.State, JobState.Queued) .SetProperty(entry => entry.CancellationRequested, false) .SetProperty(entry => entry.AttemptToken, (Guid?)null) .SetProperty(entry => entry.Progress, 0).SetProperty(entry => entry.ProgressMessage, (string?)null) .SetProperty(entry => entry.FinishedAtUtcTicks, (long?)null) .SetProperty(entry => entry.ErrorCode, (string?)null).SetProperty(entry => entry.ResultReference, (string?)null) .SetProperty(entry => entry.NextRunAtUtcTicks, next).SetProperty(entry => entry.UpdatedAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); return changed == 1 ? JobUpdateResult.Applied : await query.AnyAsync(cancellationToken) ? JobUpdateResult.Rejected : JobUpdateResult.NotFound; }); } public Task PruneAsync(string scope, JobState terminalState, DateTimeOffset finishedBeforeUtc, int take = 100, CancellationToken cancellationToken = default) { var hash = ScopeHash(scope); ValidateTake(take); if (terminalState is not (JobState.Succeeded or JobState.Failed or JobState.Cancelled or JobState.Interrupted)) throw new ArgumentOutOfRangeException(nameof(terminalState), "Retention requires a terminal job state."); var cutoff = finishedBeforeUtc.UtcTicks; if (cutoff > clock.GetUtcNow().UtcTicks) throw new ArgumentOutOfRangeException(nameof(finishedBeforeUtc), "The retention cutoff cannot be in the future."); return IndependentlyAsync(async context => { var eligible = Jobs(context).Where(entry => entry.ScopeHash == hash && entry.State == terminalState && entry.FinishedAtUtcTicks < cutoff); var candidates = await eligible.OrderBy(entry => entry.FinishedAtUtcTicks).ThenBy(entry => entry.Id) .Take(take).Select(entry => new { entry.Id, entry.Version }).ToListAsync(cancellationToken); var deleted = 0; foreach (var candidate in candidates) { // Retry changes the version and eligibility; recheck both in the deletion itself. deleted += await eligible.Where(entry => entry.Id == candidate.Id && entry.Version == candidate.Version) .ExecuteDeleteAsync(cancellationToken); } return deleted; }); } private Task FinishAsync(JobAttemptToken attempt, JobState finalState, string? errorCode, string? resultReference, CancellationToken cancellationToken, bool requeueInterrupted = false) { ValidateAttempt(attempt); return IndependentlyAsync(async context => { var now = clock.GetUtcNow().UtcTicks; await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); // Evaluate cancellation in the same UPDATE that publishes success: the first committed transition wins. var changed = await Owned(context, attempt).ExecuteUpdateAsync(setters => setters .SetProperty(entry => entry.State, entry => entry.CancellationRequested ? JobState.Cancelled : finalState) .SetProperty(entry => entry.Progress, entry => !entry.CancellationRequested && finalState == JobState.Succeeded ? 100 : entry.Progress) .SetProperty(entry => entry.ErrorCode, entry => entry.CancellationRequested ? null : errorCode) .SetProperty(entry => entry.ResultReference, entry => entry.CancellationRequested ? null : resultReference) .SetProperty(entry => entry.AttemptToken, (Guid?)null) .SetProperty(entry => entry.FinishedAtUtcTicks, now).SetProperty(entry => entry.UpdatedAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); if (changed == 0) return false; var job = await FindAsync(context, attempt.Scope, attempt.JobId, cancellationToken) ?? throw new InvalidOperationException("A completed job disappeared inside its transaction."); var updated = await CurrentAttempt(context, attempt).ExecuteUpdateAsync(setters => setters .SetProperty(entry => entry.State, job.State).SetProperty(entry => entry.Progress, job.Progress) .SetProperty(entry => entry.ProgressMessage, job.ProgressMessage).SetProperty(entry => entry.FinishedAtUtcTicks, now) .SetProperty(entry => entry.ErrorCode, job.ErrorCode).SetProperty(entry => entry.ResultReference, job.ResultReference), cancellationToken); RequireAttempt(updated); if (requeueInterrupted && job.State == JobState.Interrupted && job.AttemptCount < job.MaximumAttempts) { var hash = ScopeHash(attempt.Scope); await Jobs(context).Where(entry => entry.ScopeHash == hash && entry.Id == attempt.JobId && entry.State == JobState.Interrupted).ExecuteUpdateAsync(setters => setters .SetProperty(entry => entry.State, JobState.Queued).SetProperty(entry => entry.Progress, 0) .SetProperty(entry => entry.ProgressMessage, (string?)null).SetProperty(entry => entry.ErrorCode, (string?)null) .SetProperty(entry => entry.FinishedAtUtcTicks, (long?)null).SetProperty(entry => entry.NextRunAtUtcTicks, now) .SetProperty(entry => entry.Version, entry => entry.Version + 1), cancellationToken); } await transaction.CommitAsync(cancellationToken); return true; }); } private async Task IndependentlyAsync(Func> operation) { if (stagingContext.Database.CurrentTransaction is not null || Transaction.Current is not null) throw new InvalidOperationException("Independent job operations cannot join an external transaction; stage submissions instead."); await using var scope = scopes.CreateAsyncScope(); var context = scope.ServiceProvider.GetRequiredService(); if (ReferenceEquals(context, stagingContext) || context.Database.CurrentTransaction is not null || context.ChangeTracker.HasChanges()) throw new InvalidOperationException("Register a clean scoped DbContext so independent job operations cannot commit unrelated changes."); EnsureModel(context); return await operation(context); } private JobEntry NewEntry(JobSubmission submission) { var now = clock.GetUtcNow().UtcTicks; return new JobEntry { ScopeHash = ScopeHash(submission.Scope), Scope = submission.Scope, Id = submission.JobId, HandlerName = submission.HandlerName, HandlerVersion = submission.HandlerVersion, PayloadJson = submission.PayloadJson, Queue = submission.Queue, MaximumAttempts = submission.MaximumAttempts, InitialNotBeforeUtcTicks = submission.NotBeforeUtc?.UtcTicks, CreatedAtUtcTicks = now, UpdatedAtUtcTicks = now, NextRunAtUtcTicks = submission.NotBeforeUtc?.UtcTicks ?? now }; } private static JobSnapshot Match(JobEntry existing, JobSubmission submission) { if (existing.Scope != submission.Scope || existing.HandlerName != submission.HandlerName || existing.HandlerVersion != submission.HandlerVersion || existing.PayloadJson != submission.PayloadJson || existing.Queue != submission.Queue || existing.MaximumAttempts != submission.MaximumAttempts || existing.InitialNotBeforeUtcTicks != submission.NotBeforeUtc?.UtcTicks) throw new JobSubmissionConflictException(submission.JobId); return existing.Snapshot(); } private static IQueryable Jobs(TContext context) => context.Set().AsNoTracking(); private static Task FindAsync(TContext context, string scope, Guid id, CancellationToken token) { var hash = Identity(scope, id); return Jobs(context).SingleOrDefaultAsync(entry => entry.ScopeHash == hash && entry.Id == id, token); } private static IQueryable Owned(TContext context, JobAttemptToken attempt) { var hash = ScopeHash(attempt.Scope); return Jobs(context).Where(entry => entry.ScopeHash == hash && entry.Id == attempt.JobId && entry.State == JobState.Running && entry.AttemptToken == attempt.Token); } private static IQueryable CurrentAttempt(TContext context, JobAttemptToken attempt) { var hash = ScopeHash(attempt.Scope); return context.Set().Where(entry => entry.ScopeHash == hash && entry.JobId == attempt.JobId && entry.Token == attempt.Token && entry.State == JobState.Running); } private static string ScopeHash(string scope) { ArgumentException.ThrowIfNullOrWhiteSpace(scope); if (scope.Length > 256) throw new ArgumentOutOfRangeException(nameof(scope)); return Convert.ToHexString(SHA256.HashData(StrictUtf8.GetBytes(scope))); } private static string Identity(string scope, Guid id) { if (id == Guid.Empty) throw new ArgumentException("A job requires a nonempty identity.", nameof(id)); return ScopeHash(scope); } private static void ValidateAttempt(JobAttemptToken attempt) { _ = Identity(attempt.Scope, attempt.JobId); if (attempt.Token == Guid.Empty) throw new ArgumentException("An execution requires a nonempty token.", nameof(attempt)); } private static void ValidateTake(int take) { if (take is < 1 or > 1000) throw new ArgumentOutOfRangeException(nameof(take)); } private static void EnsureModel(TContext context) { if (context.Model.FindEntityType(typeof(JobEntry)) is null || context.Model.FindEntityType(typeof(JobAttemptEntry)) is null) throw new InvalidOperationException("Call AddTaskCenter in the host DbContext model before using the job store."); } private static void RequireAttempt(int updated) { if (updated != 1) throw new InvalidOperationException("The current job and its attempt history are inconsistent."); } }