feat(firebaseai): add bidi transcript (#17700)

* Add transcription into server content * Add transcription config * transcription working * add test for transcription * apply gemini suggestion * fix the transcript index in bidi_page * more review comment

Author: cynthiajoan

packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart

@@ -57,6 +57,8 @@ class _BidiPageState extends State<BidiPage> {
  StreamController<bool> _stopController = StreamController<bool>();
  final AudioOutput _audioOutput = AudioOutput();
  final AudioInput _audioInput = AudioInput();
+ int? _inputTranscriptionMessageIndex;
+ int? _outputTranscriptionMessageIndex;
 
  @override
  void initState() {
@@ -67,6 +69,8 @@ class _BidiPageState extends State<BidiPage> {
  responseModalities: [
  ResponseModalities.audio,
  ],
+ inputAudioTranscription: AudioTranscriptionConfig(),
+ outputAudioTranscription: AudioTranscriptionConfig(),
  );
 
  // ignore: deprecated_member_use
@@ -353,6 +357,49 @@ class _BidiPageState extends State<BidiPage> {
  if (message.modelTurn != null) {
  await _handleLiveServerContent(message);
  }
+
+ int? _handleTranscription(
+ Transcription? transcription,
+ int? messageIndex,
+ String prefix,
+ bool fromUser,
+ ) {
+ int? currentIndex = messageIndex;
+ if (transcription?.text != null) {
+ if (currentIndex != null) {
+ _messages[currentIndex] = _messages[currentIndex].copyWith(
+ text: '${_messages[currentIndex].text}${transcription!.text!}',
+ );
+ } else {
+ _messages.add(
+ MessageData(
+ text: '$prefix${transcription!.text!}',
+ fromUser: fromUser,
+ ),
+ );
+ currentIndex = _messages.length - 1;
+ }
+ if (transcription.finished ?? false) {
+ currentIndex = null;
+ }
+ setState(_scrollDown);
+ }
+ return currentIndex;
+ }
+
+ _inputTranscriptionMessageIndex = _handleTranscription(
+ message.inputTranscription,
+ _inputTranscriptionMessageIndex,
+ 'Input transcription: ',
+ true,
+ );
+ _outputTranscriptionMessageIndex = _handleTranscription(
+ message.outputTranscription,
+ _outputTranscriptionMessageIndex,
+ 'Output transcription: ',
+ false,
+ );
+
  if (message.interrupted != null && message.interrupted!) {
  developer.log('Interrupted: $response');
  }

packages/firebase_ai/firebase_ai/example/lib/widgets/message_widget.dart

@@ -22,6 +22,21 @@ class MessageData {
  this.fromUser,
  this.isThought = false,
  });
+
+ MessageData copyWith({
+ Uint8List? imageBytes,
+ String? text,
+ bool? fromUser,
+ bool? isThought,
+ }) {
+ return MessageData(
+ imageBytes: imageBytes ?? this.imageBytes,
+ text: text ?? this.text,
+ fromUser: fromUser ?? this.fromUser,
+ isThought: isThought ?? this.isThought,
+ );
+ }
+
  final Uint8List? imageBytes;
  final String? text;
  final bool? fromUser;

packages/firebase_ai/firebase_ai/lib/firebase_ai.dart

@@ -93,11 +93,13 @@ export 'src/live_api.dart'
  show
  LiveGenerationConfig,
  SpeechConfig,
+ AudioTranscriptionConfig,
  LiveServerMessage,
  LiveServerContent,
  LiveServerToolCall,
  LiveServerToolCallCancellation,
- LiveServerResponse;
+ LiveServerResponse,
+ Transcription;
 export 'src/live_session.dart' show LiveSession;
 export 'src/schema.dart' show Schema, SchemaType;
 export 'src/tool.dart'

packages/firebase_ai/firebase_ai/lib/src/live_api.dart

@@ -71,11 +71,19 @@ class SpeechConfig {
  };
 }
 
+/// The audio transcription configuration.
+class AudioTranscriptionConfig {
+ // ignore: public_member_api_docs
+ Map<String, Object?> toJson() => {};
+}
+
 /// Configures live generation settings.
 final class LiveGenerationConfig extends BaseGenerationConfig {
  // ignore: public_member_api_docs
  LiveGenerationConfig({
  this.speechConfig,
+ this.inputAudioTranscription,
+ this.outputAudioTranscription,
  super.responseModalities,
  super.maxOutputTokens,
  super.temperature,
@@ -88,6 +96,13 @@ final class LiveGenerationConfig extends BaseGenerationConfig {
  /// The speech configuration.
  final SpeechConfig? speechConfig;
 
+ /// The transcription of the input aligns with the input audio language.
+ final AudioTranscriptionConfig? inputAudioTranscription;
+
+ /// The transcription of the output aligns with the language code specified for
+ /// the output audio.
+ final AudioTranscriptionConfig? outputAudioTranscription;
+
  @override
  Map<String, Object?> toJson() => {
  ...super.toJson(),
@@ -109,14 +124,33 @@ sealed class LiveServerMessage {}
 /// with the live server has finished successfully.
 class LiveServerSetupComplete implements LiveServerMessage {}
 
+/// Audio transcription message.
+class Transcription {
+ // ignore: public_member_api_docs
+ const Transcription({this.text, this.finished});
+
+ /// Transcription text.
+ final String? text;
+
+ /// Whether this is the end of the transcription.
+ final bool? finished;
+}
+
 /// Content generated by the model in a live stream.
 class LiveServerContent implements LiveServerMessage {
  /// Creates a [LiveServerContent] instance.
  ///
  /// [modelTurn] (optional): The content generated by the model.
  /// [turnComplete] (optional): Indicates if the turn is complete.
  /// [interrupted] (optional): Indicates if the generation was interrupted.
- LiveServerContent({this.modelTurn, this.turnComplete, this.interrupted});
+ /// [inputTranscription] (optional): The input transcription.
+ /// [outputTranscription] (optional): The output transcription.
+ LiveServerContent(
+ {this.modelTurn,
+ this.turnComplete,
+ this.interrupted,
+ this.inputTranscription,
+ this.outputTranscription});
 
  // TODO(cynthia): Add accessor for media content
  /// The content generated by the model.
@@ -129,6 +163,18 @@ class LiveServerContent implements LiveServerMessage {
  /// Whether generation was interrupted. If true, indicates that a
  /// client message has interrupted current model
  final bool? interrupted;
+
+ /// The input transcription.
+ ///
+ /// The transcription is independent to the model turn which means it doesn't
+ /// imply any ordering between transcription and model turn.
+ final Transcription? inputTranscription;
+
+ /// The output transcription.
+ ///
+ /// The transcription is independent to the model turn which means it doesn't
+ /// imply any ordering between transcription and model turn.
+ final Transcription? outputTranscription;
 }
 
 /// A tool call in a live stream.
@@ -344,7 +390,26 @@ LiveServerMessage _parseServerMessage(Object jsonObject) {
  if (serverContentJson.containsKey('turnComplete')) {
  turnComplete = serverContentJson['turnComplete'] as bool;
  }
- return LiveServerContent(modelTurn: modelTurn, turnComplete: turnComplete);
+ final interrupted = serverContentJson['interrupted'] as bool?;
+ Transcription? _parseTranscription(String key) {
+ if (serverContentJson.containsKey(key)) {
+ final transcriptionJson =
+ serverContentJson[key] as Map<String, dynamic>;
+ return Transcription(
+ text: transcriptionJson['text'] as String?,
+ finished: transcriptionJson['finished'] as bool?,
+ );
+ }
+ return null;
+ }
+
+ return LiveServerContent(
+ modelTurn: modelTurn,
+ turnComplete: turnComplete,
+ interrupted: interrupted,
+ inputTranscription: _parseTranscription('inputTranscription'),
+ outputTranscription: _parseTranscription('outputTranscription'),
+ );
  } else if (json.containsKey('toolCall')) {
  final toolContentJson = json['toolCall'] as Map<String, dynamic>;
  List<FunctionCall> functionCalls = [];

packages/firebase_ai/firebase_ai/lib/src/live_model.dart

@@ -101,11 +101,18 @@ final class LiveGenerativeModel extends BaseModel {
  final setupJson = {
  'setup': {
  'model': modelString,
- if (_liveGenerationConfig != null)
- 'generation_config': _liveGenerationConfig.toJson(),
  if (_systemInstruction != null)
  'system_instruction': _systemInstruction.toJson(),
  if (_tools != null) 'tools': _tools.map((t) => t.toJson()).toList(),
+ if (_liveGenerationConfig != null) ...{
+ 'generation_config': _liveGenerationConfig.toJson(),
+ if (_liveGenerationConfig.inputAudioTranscription != null)
+ 'input_audio_transcription':
+ _liveGenerationConfig.inputAudioTranscription!.toJson(),
+ if (_liveGenerationConfig.outputAudioTranscription != null)
+ 'output_audio_transcription':
+ _liveGenerationConfig.outputAudioTranscription!.toJson(),
+ },
  }
  };
 

packages/firebase_ai/firebase_ai/test/live_test.dart

@@ -240,5 +240,42 @@ void main() {
  expect(() => parseServerResponse(jsonObject),
  throwsA(isA<FirebaseAISdkException>()));
  });
+
+ test(
+ 'LiveGenerationConfig with transcriptions toJson() returns correct JSON',
+ () {
+ final liveGenerationConfig = LiveGenerationConfig(
+ inputAudioTranscription: AudioTranscriptionConfig(),
+ outputAudioTranscription: AudioTranscriptionConfig(),
+ );
+ // Explicitly, these two config should not exist in the toJson()
+ expect(liveGenerationConfig.toJson(), {});
+ });
+
+ test('parseServerMessage parses serverContent with transcriptions', () {
+ final jsonObject = {
+ 'serverContent': {
+ 'modelTurn': {
+ 'parts': [
+ {'text': 'Hello, world!'}
+ ]
+ },
+ 'turnComplete': true,
+ 'inputTranscription': {'text': 'input', 'finished': true},
+ 'outputTranscription': {'text': 'output', 'finished': false}
+ }
+ };
+ final response = parseServerResponse(jsonObject);
+ expect(response.message, isA<LiveServerContent>());
+ final contentMessage = response.message as LiveServerContent;
+ expect(contentMessage.turnComplete, true);
+ expect(contentMessage.modelTurn, isA<Content>());
+ expect(contentMessage.inputTranscription, isA<Transcription>());
+ expect(contentMessage.inputTranscription?.text, 'input');
+ expect(contentMessage.inputTranscription?.finished, true);
+ expect(contentMessage.outputTranscription, isA<Transcription>());
+ expect(contentMessage.outputTranscription?.text, 'output');
+ expect(contentMessage.outputTranscription?.finished, false);
+ });
  });
 }