using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using MS.Microservice.Media.Images;
using SkiaSharp;
namespace MS.Microservice.Media.Inference.SuperResolution;
/// A verified local Swin2SR CPU session for tiled twofold super-resolution. Reuse and asynchronously dispose the instance.
public sealed class Swin2SrUpscaler : IAsyncDisposable
{
public const string ModelId = "Xenova/swin2SR-lightweight-x2-64@92a21aca5713f20faf9a87590cdfbdce2e34112c";
public const string ModelSha256 = "C2ABBFE0CC8E685B5E11964970F8EBE3D24072E904FD5545A30AC31EC1E110DB";
internal const int ModelBytes = 8_078_888;
private readonly InferenceSession _session;
private readonly Swin2SrOptions _options;
private readonly SemaphoreSlim _gate = new(1, 1);
private int _disposeRequested;
private bool _disposed;
private Swin2SrUpscaler(InferenceSession session, Swin2SrOptions options) { _session = session; _options = options; }
public static async Task CreateAsync(Swin2SrOptions options, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentException.ThrowIfNullOrWhiteSpace(options.ModelPath);
ImageCodec.ValidateLimits(options.Limits);
if (options.CpuThreads is < 1 or > 64 || options.MaximumOutputDimension < 1 || options.MaximumOutputPixels < 1
|| options.TileSize is < 32 or > 256 || options.TileSize % 8 != 0
|| options.ContextPixels is < 0 or > 64 || options.ContextPixels % 8 != 0 || 2 * options.ContextPixels > options.TileSize)
throw new ArgumentException("Invalid CPU, raster or tile budgets.", nameof(options));
var bytes = await VerifiedOnnxModel.ReadAsync(options.ModelPath, ModelBytes, ModelSha256, cancellationToken).ConfigureAwait(false);
using var settings = new SessionOptions
{ IntraOpNumThreads = options.CpuThreads, InterOpNumThreads = 1, ExecutionMode = ExecutionMode.ORT_SEQUENTIAL };
var session = await Task.Run(() => new InferenceSession(bytes, settings), cancellationToken).ConfigureAwait(false);
try
{
cancellationToken.ThrowIfCancellationRequested();
if (session.InputMetadata.Count != 1 || session.OutputMetadata.Count != 1
|| !session.InputMetadata.TryGetValue("pixel_values", out var input)
|| !session.OutputMetadata.TryGetValue("reconstruction", out var output)
|| !FloatNchw(input) || !FloatNchw(output))
throw new InvalidDataException("The supported model requires float NCHW input and reconstruction.");
return new(session, options with { Limits = options.Limits with { } });
}
catch { session.Dispose(); throw; }
}
/// Writes a twofold PNG with bilinearly scaled source alpha; streams remain open. Failure or cancellation may leave partial output.
public async Task UpscaleAsync(Stream inputImage, Stream outputPng, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(inputImage);
ArgumentNullException.ThrowIfNull(outputPng);
if (ReferenceEquals(inputImage, outputPng) || !inputImage.CanRead || !outputPng.CanWrite)
throw new ArgumentException("Use distinct readable input and writable output streams.");
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeRequested) != 0, this);
using var input = await ImageCodec.DecodeAsync(inputImage, _options.Limits, cancellationToken).ConfigureAwait(false);
using var output = ImageCodec.CreateBitmap(checked(input.Width * 2), checked(input.Height * 2),
new ImageLimits { MaximumDimension = _options.MaximumOutputDimension, MaximumPixels = _options.MaximumOutputPixels });
var tiles = 0;
for (var y = 0; y < input.Height; y += _options.TileSize)
for (var x = 0; x < input.Width; x += _options.TileSize)
{
cancellationToken.ThrowIfCancellationRequested();
var width = Math.Min(_options.TileSize, input.Width - x);
var height = Math.Min(_options.TileSize, input.Height - y);
var left = Math.Max(0, x - _options.ContextPixels);
var top = Math.Max(0, y - _options.ContextPixels);
var regionWidth = Math.Min(input.Width, x + width + _options.ContextPixels) - left;
var regionHeight = Math.Min(input.Height, y + height + _options.ContextPixels) - top;
var paddedWidth = Math.Max(8, (regionWidth + 7) / 8 * 8);
var paddedHeight = Math.Max(8, (regionHeight + 7) / 8 * 8);
var tensor = CreateTensor(input, left, top, regionWidth, regionHeight, paddedWidth, paddedHeight, cancellationToken);
using var inputValue = OrtValue.CreateTensorValueFromMemory(tensor, [1, 3, paddedHeight, paddedWidth]);
using var runOptions = new RunOptions();
using var cancel = cancellationToken.Register(static state => ((RunOptions)state!).Terminate = true, runOptions);
IDisposableReadOnlyCollection results;
try
{
results = await Task.Run(() => _session.Run(runOptions, ["pixel_values"], [inputValue], ["reconstruction"])).ConfigureAwait(false);
}
catch (OnnxRuntimeException) when (cancellationToken.IsCancellationRequested)
{ throw new OperationCanceledException(cancellationToken); }
using (results)
{
cancellationToken.ThrowIfCancellationRequested();
var value = results.Single();
var shape = value.GetTensorTypeAndShape();
if (shape.ElementDataType != TensorElementType.Float
|| !shape.Shape.SequenceEqual(new long[] { 1, 3, paddedHeight * 2, paddedWidth * 2 }))
throw new InvalidDataException("The model returned an unexpected twofold raster.");
Compose(input, output, value.GetTensorDataAsSpan(), x, y, width, height,
(x - left) * 2, (y - top) * 2, paddedWidth * 2, paddedHeight * 2, cancellationToken);
}
tiles++;
}
await ImageCodec.EncodeAsync(output, outputPng, ImageOutputFormat.Png, 100, new(255, 255, 255), cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return new(input.Width, input.Height, output.Width, output.Height, tiles, ModelId);
}
finally { _gate.Release(); }
}
public async ValueTask DisposeAsync()
{
Interlocked.Exchange(ref _disposeRequested, 1);
await _gate.WaitAsync().ConfigureAwait(false);
try { if (!_disposed) { _disposed = true; _session.Dispose(); } }
finally { _gate.Release(); }
}
private static bool FloatNchw(NodeMetadata value) => value.IsTensor && value.ElementType == typeof(float) && value.Dimensions.Length == 4;
internal static int Reflect(int index, int length)
{
if (length == 1) return 0;
var period = 2 * (length - 1);
var remainder = index % period;
return remainder < length ? remainder : period - remainder;
}
private static float[] CreateTensor(SKBitmap source, int left, int top, int width, int height, int paddedWidth, int paddedHeight, CancellationToken token)
{
var plane = paddedWidth * paddedHeight;
var tensor = new float[plane * 3];
for (var y = 0; y < paddedHeight; y++)
{
token.ThrowIfCancellationRequested();
for (var x = 0; x < paddedWidth; x++)
{
var pixel = source.GetPixel(left + Reflect(x, width), top + Reflect(y, height));
var alpha = pixel.Alpha / 255f;
var index = y * paddedWidth + x;
tensor[index] = pixel.Red / 255f * alpha + 1 - alpha;
tensor[plane + index] = pixel.Green / 255f * alpha + 1 - alpha;
tensor[2 * plane + index] = pixel.Blue / 255f * alpha + 1 - alpha;
}
}
return tensor;
}
private static void Compose(SKBitmap source, SKBitmap output, ReadOnlySpan rgb, int x, int y, int width, int height,
int cropX, int cropY, int rasterWidth, int rasterHeight, CancellationToken token)
{
foreach (var value in rgb)
if (!float.IsFinite(value)) throw new InvalidDataException("The model returned a non-finite pixel.");
var plane = rasterWidth * rasterHeight;
for (var row = 0; row < height * 2; row++)
{
token.ThrowIfCancellationRequested();
for (var column = 0; column < width * 2; column++)
{
var index = (cropY + row) * rasterWidth + cropX + column;
var outputX = x * 2 + column;
var outputY = y * 2 + row;
output.SetPixel(outputX, outputY, new(ToByte(rgb[index]), ToByte(rgb[plane + index]), ToByte(rgb[2 * plane + index]),
ScaleAlpha(source, outputX, outputY)));
}
}
}
private static byte ToByte(float value) => (byte)Math.Clamp((int)MathF.Round(Math.Clamp(value, 0, 1) * 255), 0, 255);
internal static byte ScaleAlpha(SKBitmap source, int x, int y)
{
var sourceX = Math.Clamp((x + 0.5) / 2 - 0.5, 0, source.Width - 1);
var sourceY = Math.Clamp((y + 0.5) / 2 - 0.5, 0, source.Height - 1);
var left = (int)sourceX;
var top = (int)sourceY;
var right = Math.Min(source.Width - 1, left + 1);
var bottom = Math.Min(source.Height - 1, top + 1);
var dx = sourceX - left;
var dy = sourceY - top;
var upper = source.GetPixel(left, top).Alpha * (1 - dx) + source.GetPixel(right, top).Alpha * dx;
var lower = source.GetPixel(left, bottom).Alpha * (1 - dx) + source.GetPixel(right, bottom).Alpha * dx;
return (byte)Math.Clamp((int)Math.Round(upper * (1 - dy) + lower * dy), 0, 255);
}
}