| File: OpenAISpeechToTextClient.cs | |
| Project: ..\..\..\src\Libraries\Microsoft.Extensions.AI.OpenAI\Microsoft.Extensions.AI.OpenAI.csproj (Microsoft.Extensions.AI.OpenAI) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; using OpenAI; using OpenAI.Audio; #pragma warning disable MEAI001 // Type is for evaluation purposes only #pragma warning disable OPENAI001 // Streaming transcription segment updates are experimental #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields #pragma warning disable SA1204 // Static elements should appear before instance elements namespace Microsoft.Extensions.AI; /// <summary>Represents an <see cref="ISpeechToTextClient"/> for an OpenAI <see cref="OpenAIClient"/> or <see cref="OpenAI.Audio.AudioClient"/>.</summary> [Experimental(DiagnosticIds.Experiments.AISpeechToText, UrlFormat = DiagnosticIds.UrlFormat)] internal sealed class OpenAISpeechToTextClient : ISpeechToTextClient { /// <summary>Metadata about the client.</summary> private readonly SpeechToTextClientMetadata _metadata; /// <summary>The underlying <see cref="AudioClient" />.</summary> private readonly AudioClient _audioClient; /// <summary>Initializes a new instance of the <see cref="OpenAISpeechToTextClient"/> class for the specified <see cref="AudioClient"/>.</summary> /// <param name="audioClient">The underlying client.</param> public OpenAISpeechToTextClient(AudioClient audioClient) { _audioClient = Throw.IfNull(audioClient); _metadata = new("openai", audioClient.Endpoint, _audioClient.Model); } /// <inheritdoc /> public object? GetService(Type serviceType, object? serviceKey = null) { _ = Throw.IfNull(serviceType); return serviceKey is not null ? null : serviceType == typeof(SpeechToTextClientMetadata) ? _metadata : serviceType == typeof(AudioClient) ? _audioClient : serviceType.IsInstanceOfType(this) ? this : null; } /// <inheritdoc /> public async Task<SpeechToTextResponse> GetTextAsync( Stream audioSpeechStream, SpeechToTextOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(audioSpeechStream); SpeechToTextResponse response = new(); string filename = ResolveFilename(audioSpeechStream); if (IsTranslationRequest(options)) { var translation = (await _audioClient.TranslateAudioAsync(audioSpeechStream, filename, ToOpenAITranslationOptions(options), cancellationToken).ConfigureAwait(false)).Value; response.Contents = [new TextContent(translation.Text)]; response.RawRepresentation = translation; int segmentCount = translation.Segments.Count; if (segmentCount > 0) { response.StartTime = translation.Segments[0].StartTime; response.EndTime = translation.Segments[segmentCount - 1].EndTime; } } else { var transcription = (await _audioClient.TranscribeAudioAsync(audioSpeechStream, filename, ToOpenAITranscriptionOptions(options), cancellationToken).ConfigureAwait(false)).Value; response.Contents = [new TextContent(transcription.Text)]; response.RawRepresentation = transcription; int segmentCount = transcription.Segments.Count; if (segmentCount > 0) { response.StartTime = transcription.Segments[0].StartTime; response.EndTime = transcription.Segments[segmentCount - 1].EndTime; } else { int wordCount = transcription.Words.Count; if (wordCount > 0) { response.StartTime = transcription.Words[0].StartTime; response.EndTime = transcription.Words[wordCount - 1].EndTime; } } if (transcription.Usage is AudioTranscriptionTokenUsage tokenUsage) { response.Usage = ToUsageDetails(tokenUsage); } } return response; } /// <inheritdoc /> public async IAsyncEnumerable<SpeechToTextResponseUpdate> GetStreamingTextAsync( Stream audioSpeechStream, SpeechToTextOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { _ = Throw.IfNull(audioSpeechStream); string filename = ResolveFilename(audioSpeechStream); if (IsTranslationRequest(options)) { foreach (var update in (await GetTextAsync(audioSpeechStream, options, cancellationToken).ConfigureAwait(false)).ToSpeechToTextResponseUpdates()) { yield return update; } } else { await foreach (var update in _audioClient.TranscribeAudioStreamingAsync( audioSpeechStream, filename, ToOpenAITranscriptionOptions(options), cancellationToken).ConfigureAwait(false)) { SpeechToTextResponseUpdate result = new() { ModelId = options?.ModelId, RawRepresentation = update, }; switch (update) { case StreamingAudioTranscriptionTextDeltaUpdate deltaUpdate: result.Kind = SpeechToTextResponseUpdateKind.TextUpdated; result.Contents = [new TextContent(deltaUpdate.Delta)]; break; case StreamingAudioTranscriptionTextSegmentUpdate segmentUpdate: result.Kind = SpeechToTextResponseUpdateKind.TextUpdated; result.StartTime = segmentUpdate.StartTime; result.EndTime = segmentUpdate.EndTime; break; case StreamingAudioTranscriptionTextDoneUpdate doneUpdate: result.Kind = SpeechToTextResponseUpdateKind.SessionClose; if (doneUpdate.Usage is { } usage) { result.Contents = [new UsageContent(ToUsageDetails(usage))]; } break; } yield return result; } } } /// <inheritdoc /> void IDisposable.Dispose() { // Nothing to dispose. Implementation required for the IAudioTranscriptionClient interface. } // <summary>A translation is triggered when the target text language is specified and the source language is not provided or different.</summary> private static bool IsTranslationRequest(SpeechToTextOptions? options) => options is not null && options.TextLanguage is not null && (options.SpeechLanguage is null || options.SpeechLanguage != options.TextLanguage); /// <summary> /// Resolves the filename to use for the audio stream in the multipart request. /// Priority: <see cref="FileStream"/> name, then magic-byte detection (seekable streams only), then default. /// </summary> private static string ResolveFilename(Stream audioSpeechStream) { const int FormatDetectionByteCount = 12; if (audioSpeechStream is FileStream fileStream) { return Path.GetFileName(fileStream.Name); } // For seekable streams positioned at the start, peek at the header to detect audio format, then rewind. if (audioSpeechStream.CanSeek && audioSpeechStream.Position == 0) { byte[] header = new byte[FormatDetectionByteCount]; int bytesRead = 0; while (bytesRead < header.Length) { int n = audioSpeechStream.Read(header, bytesRead, header.Length - bytesRead); if (n <= 0) { break; } bytesRead += n; } audioSpeechStream.Position -= bytesRead; return $"audio.{DetectAudioExtension(header.AsSpan(0, bytesRead))}"; } return "audio.mp3"; } /// <summary>Detects the audio format extension from the leading bytes of the audio data.</summary> private static string DetectAudioExtension(ReadOnlySpan<byte> header) { // WAV: "RIFF" at offset 0 and "WAVE" at offset 8. if (header.Length >= 12 && header.Slice(0, 4).SequenceEqual("RIFF"u8) && header.Slice(8, 4).SequenceEqual("WAVE"u8)) { return "wav"; } // WebM/Matroska: EBML header ID at offset 0. if (header.Length >= 4 && header.Slice(0, 4).SequenceEqual((ReadOnlySpan<byte>)[0x1A, 0x45, 0xDF, 0xA3])) { return "webm"; } // M4A/MP4: ISO BMFF "ftyp" box type at offset 4. if (header.Length >= 8 && header.Slice(4, 4).SequenceEqual("ftyp"u8)) { return "m4a"; } // MP3: ID3v2 tag at offset 0. if (header.Length >= 3 && header.Slice(0, 3).SequenceEqual("ID3"u8)) { return "mp3"; } // MP3: MPEG frame sync word (11 set bits). if (header.Length >= 2 && header[0] == 0xFF && (header[1] & 0xE0) == 0xE0) { return "mp3"; } return "mp3"; } /// <summary>Converts an extensions options instance to an OpenAI transcription options instance.</summary> private AudioTranscriptionOptions ToOpenAITranscriptionOptions(SpeechToTextOptions? options) { AudioTranscriptionOptions result = options?.RawRepresentationFactory?.Invoke(this) as AudioTranscriptionOptions ?? new(); result.Language ??= options?.SpeechLanguage; return result; } /// <summary>Converts an extensions options instance to an OpenAI translation options instance.</summary> private AudioTranslationOptions ToOpenAITranslationOptions(SpeechToTextOptions? options) { AudioTranslationOptions result = options?.RawRepresentationFactory?.Invoke(this) as AudioTranslationOptions ?? new(); return result; } /// <summary>Maps <see cref="AudioTranscriptionTokenUsage"/> to <see cref="UsageDetails"/>.</summary> private static UsageDetails ToUsageDetails(AudioTranscriptionTokenUsage tokenUsage) { var details = new UsageDetails { InputTokenCount = tokenUsage.InputTokenCount, OutputTokenCount = tokenUsage.OutputTokenCount, TotalTokenCount = tokenUsage.TotalTokenCount, }; if (tokenUsage.InputTokenDetails is { } inputDetails) { details.InputAudioTokenCount = inputDetails.AudioTokenCount; details.InputTextTokenCount = inputDetails.TextTokenCount; } return details; } }