| File: OpenAITextToSpeechClient.cs | |
| Project: ..\..\..\src\Libraries\Microsoft.Extensions.AI.OpenAI\Microsoft.Extensions.AI.OpenAI.csproj (Microsoft.Extensions.AI.OpenAI) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; using System.ClientModel; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; using OpenAI; using OpenAI.Audio; #pragma warning disable OPENAI001 // Streaming speech generation is experimental #pragma warning disable SA1204 // Static elements should appear before instance elements namespace Microsoft.Extensions.AI; /// <summary>Represents an <see cref="ITextToSpeechClient"/> for an OpenAI <see cref="OpenAIClient"/> or <see cref="AudioClient"/>.</summary> [Experimental(DiagnosticIds.Experiments.AITextToSpeech, UrlFormat = DiagnosticIds.UrlFormat)] internal sealed class OpenAITextToSpeechClient : ITextToSpeechClient { /// <summary>Default voice to use when none is specified.</summary> private const string DefaultVoice = "alloy"; /// <summary>Metadata about the client.</summary> private readonly TextToSpeechClientMetadata _metadata; /// <summary>The underlying <see cref="AudioClient" />.</summary> private readonly AudioClient _audioClient; /// <summary>Initializes a new instance of the <see cref="OpenAITextToSpeechClient"/> class for the specified <see cref="AudioClient"/>.</summary> /// <param name="audioClient">The underlying client.</param> public OpenAITextToSpeechClient(AudioClient audioClient) { _audioClient = Throw.IfNull(audioClient); _metadata = new("openai", audioClient.Endpoint, _audioClient.Model); } /// <inheritdoc /> public object? GetService(Type serviceType, object? serviceKey = null) { _ = Throw.IfNull(serviceType); return serviceKey is not null ? null : serviceType == typeof(TextToSpeechClientMetadata) ? _metadata : serviceType == typeof(AudioClient) ? _audioClient : serviceType.IsInstanceOfType(this) ? this : null; } /// <inheritdoc /> public async Task<TextToSpeechResponse> GetAudioAsync( string text, TextToSpeechOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(text); SpeechGenerationOptions openAIOptions = ToOpenAISpeechOptions(options); var result = await _audioClient.GenerateSpeechAsync( text, new GeneratedSpeechVoice(options?.VoiceId ?? DefaultVoice), openAIOptions, cancellationToken).ConfigureAwait(false); string mediaType = GetMediaType(openAIOptions.ResponseFormat); return new TextToSpeechResponse([new DataContent(result.Value.ToMemory(), mediaType)]) { ModelId = options?.ModelId ?? _metadata.DefaultModelId, RawRepresentation = result, }; } /// <inheritdoc /> public async IAsyncEnumerable<TextToSpeechResponseUpdate> GetStreamingAudioAsync( string text, TextToSpeechOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { _ = Throw.IfNull(text); SpeechGenerationOptions openAIOptions = ToOpenAISpeechOptions(options); string mediaType = GetMediaType(openAIOptions.ResponseFormat); AsyncCollectionResult<StreamingSpeechUpdate>? streamingResult = null; try { streamingResult = _audioClient.GenerateSpeechStreamingAsync( text, new GeneratedSpeechVoice(options?.VoiceId ?? DefaultVoice), openAIOptions, cancellationToken); } catch (NotSupportedException) { // Model doesn't support SSE streaming (e.g. tts-1, tts-1-hd). } if (streamingResult is null) { // Fall back to non-streaming for models that don't support SSE streaming. foreach (var update in (await GetAudioAsync(text, options, cancellationToken).ConfigureAwait(false)).ToTextToSpeechResponseUpdates()) { yield return update; } yield break; } await foreach (var update in streamingResult.ConfigureAwait(false)) { switch (update) { case StreamingSpeechAudioDeltaUpdate deltaUpdate: yield return new TextToSpeechResponseUpdate { Kind = TextToSpeechResponseUpdateKind.AudioUpdating, Contents = [new DataContent(deltaUpdate.AudioBytes.ToMemory(), mediaType)], ModelId = options?.ModelId ?? _metadata.DefaultModelId, RawRepresentation = deltaUpdate, }; break; case StreamingSpeechAudioDoneUpdate doneUpdate: var sessionClose = new TextToSpeechResponseUpdate { Kind = TextToSpeechResponseUpdateKind.SessionClose, ModelId = options?.ModelId ?? _metadata.DefaultModelId, RawRepresentation = doneUpdate, }; if (doneUpdate.Usage is { } usage) { sessionClose.Contents = [new UsageContent(new() { InputTokenCount = usage.InputTokenCount, OutputTokenCount = usage.OutputTokenCount, TotalTokenCount = usage.TotalTokenCount, })]; } yield return sessionClose; break; } } } /// <inheritdoc /> void IDisposable.Dispose() { // Nothing to dispose. Implementation required for the ITextToSpeechClient interface. } /// <summary>Converts an extensions options instance to an OpenAI speech generation options instance.</summary> private SpeechGenerationOptions ToOpenAISpeechOptions(TextToSpeechOptions? options) { SpeechGenerationOptions result = options?.RawRepresentationFactory?.Invoke(this) as SpeechGenerationOptions ?? new(); if (options?.Speed is float speed) { result.SpeedRatio ??= speed; } if (options?.AudioFormat is string audioFormat) { result.ResponseFormat ??= ToGeneratedSpeechFormat(audioFormat); } return result; } /// <summary>Maps a format string to a <see cref="GeneratedSpeechFormat"/>.</summary> private static GeneratedSpeechFormat? ToGeneratedSpeechFormat(string format) => format.ToUpperInvariant() switch { "MP3" or "AUDIO/MPEG" => GeneratedSpeechFormat.Mp3, "OPUS" or "AUDIO/OPUS" => GeneratedSpeechFormat.Opus, "AAC" or "AUDIO/AAC" => GeneratedSpeechFormat.Aac, "FLAC" or "AUDIO/FLAC" => GeneratedSpeechFormat.Flac, "WAV" or "AUDIO/WAV" => GeneratedSpeechFormat.Wav, "PCM" or "AUDIO/L16" => GeneratedSpeechFormat.Pcm, _ => new GeneratedSpeechFormat(format), }; /// <summary>Gets the media type for the specified response format.</summary> private static string GetMediaType(GeneratedSpeechFormat? format) => format?.ToString() switch { "mp3" => "audio/mpeg", "opus" => "audio/opus", "aac" => "audio/aac", "flac" => "audio/flac", "wav" => "audio/wav", "pcm" => "audio/l16", null => "audio/mpeg", // OpenAI default is mp3 _ => "application/octet-stream", }; }