使用 Gemini Live API 进行双向流式传输

 import FirebaseAILogic // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) let model = FirebaseAI.firebaseAI(backend: .googleAI()).liveModel(  modelName: "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with text  generationConfig: LiveGenerationConfig(  responseModalities: [.text]  ) ) do {  let session = try await model.connect()  // Load the audio file, or tap a microphone  guard let audioFile = NSDataAsset(name: "audio.pcm") else {  fatalError("Failed to load audio file")  }  // Provide the audio data  await session.sendAudioRealtime(audioFile.data)  var outputText = ""  for try await message in session.responses {  if case let .content(content) = message.payload {  content.modelTurn?.parts.forEach { part in  if let part = part as? TextPart {  outputText += part.text  }  }  // Optional: if you don't require to send more requests.  if content.isTurnComplete {  await session.close()  }  }  }  // Output received from the server.  print(outputText) } catch {  fatalError(error.localizedDescription) } 

 // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) val model = Firebase.ai(backend = GenerativeBackend.googleAI()).liveModel(  modelName = "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with text  generationConfig = liveGenerationConfig {  responseModality = ResponseModality.TEXT  } ) val session = model.connect() // Provide a text prompt val audioContent = content("user") { audioData } session.send(audioContent) var outputText = "" session.receive().collect {  if(it.status == Status.TURN_COMPLETE) {  // Optional: if you don't require to send more requests.  session.stopReceiving();  }  outputText = outputText + it.text } // Output received from the server. println(outputText) 

 ExecutorService executor = Executors.newFixedThreadPool(1); // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) LiveGenerativeModel lm = FirebaseAI.getInstance(GenerativeBackend.googleAI()).liveModel(  "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with text  new LiveGenerationConfig.Builder()  .setResponseModalities(ResponseModality.TEXT)  .build() ); LiveModelFutures model = LiveModelFutures.from(lm); ListenableFuture<LiveSession> sessionFuture = model.connect(); class LiveContentResponseSubscriber implements Subscriber<LiveContentResponse> {  @Override  public void onSubscribe(Subscription s) {  s.request(Long.MAX_VALUE); // Request an unlimited number of items  }  @Override  public void onNext(LiveContentResponse liveContentResponse) {  // Handle the response from the server. System.out.println(liveContentResponse.getText());  }  @Override  public void onError(Throwable t) {  System.err.println("Error: " + t.getMessage());  }  @Override  public void onComplete() {  System.out.println("Done receiving messages!");  } } Futures.addCallback(sessionFuture, new FutureCallback<LiveSession>() {  @Override  public void onSuccess(LiveSession ses) {  LiveSessionFutures session = LiveSessionFutures.from(ses);  // Send Audio data  session.send(new Content.Builder().addInlineData(audioData, "audio/pcm").build());  session.send(text);  Publisher<LiveContentResponse> publisher = session.receive();  publisher.subscribe(new LiveContentResponseSubscriber());  }  @Override  public void onFailure(Throwable t) {  // Handle exceptions  } }, executor); 

 // Initialize the Gemini Developer API backend service const ai = getAI(firebaseApp, { backend: new GoogleAIBackend() }); // Create a `LiveGenerativeModel` instance with the flash-live model (only model that supports the Live API) const model = getLiveGenerativeModel(ai, {  model: "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with text  generationConfig: {  responseModalities: [ResponseModality.TEXT],  }, }); const session = await model.connect(); // TODO(developer): Collect audio data (16-bit 16kHz PCM) // const audioData = ... // Send audio const audioPart = {  inlineData: { data: audioData, mimeType: "audio/pcm" }, }; session.send([audioPart]); // Collect text from model's turn let text = ""; const messages = session.receive(); for await (const message of messages) {  switch (message.type) {  case "serverContent":  if (message.turnComplete) {  console.log(text);  } else {  const parts = message.modelTurn?.parts;  if (parts) {  text += parts.map((part) => part.text).join("");  }  }  break;  case "toolCall":  // Ignore  case "toolCallCancellation":  // Ignore  } } 

 import 'package:firebase_ai/firebase_ai.dart'; import 'package:firebase_core/firebase_core.dart'; import 'firebase_options.dart'; import 'package:your_audio_recorder_package/your_audio_recorder_package.dart'; import 'dart:async'; late LiveModelSession _session; final _audioRecorder = YourAudioRecorder(); await Firebase.initializeApp(  options: DefaultFirebaseOptions.currentPlatform, ); // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) final model = FirebaseAI.googleAI().liveGenerativeModel(  model: 'gemini-2.0-flash-live-preview-04-09',  // Configure the model to respond with text  liveGenerationConfig: LiveGenerationConfig(responseModalities: ResponseModalities.text), ); _session = await model.connect(); final audioRecordStream = _audioRecorder.startRecordingStream(); final mediaChunkStream = audioRecordStream.map((data) {  return InlineDataPart('audio/pcm', data); }); await _session.startMediaStream(mediaChunkStream); final responseStream = _session.receive(); return responseStream.asyncMap((response) async {  if (response.parts.isNotEmpty && response.parts.first.text != null) {  return response.parts.first.text!;  } else {  throw Exception('Text response not found.');  } }); Future main() async {  try {  final textStream = await audioToText();  await for (final text in textStream) {  print('Received text: $text');  // Handle the text response  }  } catch (e) {  print('Error: $e');  } } 

 using Firebase; using Firebase.AI; async Task SendAudioReceiveText() {  // Initialize the Gemini Developer API backend service  // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)  var model = FirebaseAI.GetInstance(FirebaseAI.Backend.GoogleAI()).GetLiveModel(  modelName: "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with text  liveGenerationConfig: new LiveGenerationConfig(  responseModalities: new[] { ResponseModality.Text })  );  LiveSession session = await model.ConnectAsync();  // Start a coroutine to send audio from the Microphone  var recordingCoroutine = StartCoroutine(SendAudio(session));  // Receive the response  await foreach (var message in session.ReceiveAsync()) {  // Process the received message  if (!string.IsNullOrEmpty(message.Text)) {  UnityEngine.Debug.Log("Received message: " + message.Text);  }  }  StopCoroutine(recordingCoroutine); } IEnumerator SendAudio(LiveSession liveSession) {  string microphoneDeviceName = null;  int recordingFrequency = 16000;  int recordingBufferSeconds = 2;  var recordingClip = Microphone.Start(microphoneDeviceName, true,  recordingBufferSeconds, recordingFrequency);  int lastSamplePosition = 0;  while (true) {  if (!Microphone.IsRecording(microphoneDeviceName)) {  yield break;  }  int currentSamplePosition = Microphone.GetPosition(microphoneDeviceName);  if (currentSamplePosition != lastSamplePosition) {  // The Microphone uses a circular buffer, so we need to check if the  // current position wrapped around to the beginning, and handle it  // accordingly.  int sampleCount;  if (currentSamplePosition > lastSamplePosition) {  sampleCount = currentSamplePosition - lastSamplePosition;  } else {  sampleCount = recordingClip.samples - lastSamplePosition + currentSamplePosition;  }  if (sampleCount > 0) {  // Get the audio chunk  float[] samples = new float[sampleCount];  recordingClip.GetData(samples, lastSamplePosition);  // Send the data, discarding the resulting Task to avoid the warning  _ = liveSession.SendAudioAsync(samples);  lastSamplePosition = currentSamplePosition;  }  }  // Wait for a short delay before reading the next sample from the Microphone  const float MicrophoneReadDelay = 0.5f;  yield return new WaitForSeconds(MicrophoneReadDelay);  } } 

 import FirebaseAILogic // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) let model = FirebaseAI.firebaseAI(backend: .googleAI()).liveModel(  modelName: "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with audio  generationConfig: LiveGenerationConfig(  responseModalities: [.audio]  ) ) do {  let session = try await model.connect()  // Provide a text prompt  let text = "tell a short story"  await session.sendTextRealtime(text)  var outputText = ""  for try await message in session.responses {  if case let .content(content) = message.payload {  content.modelTurn?.parts.forEach { part in  if let part = part as? InlineDataPart, part.mimeType.starts(with: "audio/pcm") {  // Handle 16bit pcm audio data at 24khz  playAudio(part.data)  }  }  // Optional: if you don't require to send more requests.  if content.isTurnComplete {  await session.close()  }  }  } } catch {  fatalError(error.localizedDescription) } 

 // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) val model = Firebase.ai(backend = GenerativeBackend.googleAI()).liveModel(  modelName = "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with audio  generationConfig = liveGenerationConfig {  responseModality = ResponseModality.AUDIO  } ) val session = model.connect() // Provide a text prompt val text = "tell a short story" session.send(text) session.receive().collect {  if(it.turnComplete) {  // Optional: if you don't require to send more requests.  session.stopReceiving();  }  // Handle 16bit pcm audio data at 24khz  playAudio(it.data) } 

 ExecutorService executor = Executors.newFixedThreadPool(1); // Initialize the Gemini Developer API backend service // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API) LiveGenerativeModel lm = FirebaseAI.getInstance(GenerativeBackend.googleAI()).liveModel(  "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with text  new LiveGenerationConfig.Builder()  .setResponseModalities(ResponseModality.AUDIO)  .build() ); LiveModelFutures model = LiveModelFutures.from(lm); ListenableFuture<LiveSession> sessionFuture = model.connect(); class LiveContentResponseSubscriber implements Subscriber<LiveContentResponse> {  @Override  public void onSubscribe(Subscription s) {  s.request(Long.MAX_VALUE); // Request an unlimited number of items  }  @Override  public void onNext(LiveContentResponse liveContentResponse) {  // Handle 16bit pcm audio data at 24khz liveContentResponse.getData();  }  @Override  public void onError(Throwable t) {  System.err.println("Error: " + t.getMessage());  }  @Override  public void onComplete() {  System.out.println("Done receiving messages!");  } } Futures.addCallback(sessionFuture, new FutureCallback<LiveSession>() {  @Override  public void onSuccess(LiveSession ses) {  LiveSessionFutures session = LiveSessionFutures.from(ses);  // Provide a text prompt  String text = "tell me a short story?";  session.send(text);  Publisher<LiveContentResponse> publisher = session.receive();  publisher.subscribe(new LiveContentResponseSubscriber());  }  @Override  public void onFailure(Throwable t) {  // Handle exceptions  } }, executor); 

 // Initialize the Gemini Developer API backend service const ai = getAI(firebaseApp, { backend: new GoogleAIBackend() }); // Create a `LiveGenerativeModel` instance with the flash-live model (only model that supports the Live API) const model = getLiveGenerativeModel(ai, {  model: "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with audio  generationConfig: {  responseModalities: [ResponseModality.AUDIO],  }, }); const session = await model.connect(); // Provide a text prompt const prompt = "tell a short story"; session.send(prompt); // Handle the model's audio output const messages = session.receive(); for await (const message of messages) {  switch (message.type) {  case "serverContent":  if (message.turnComplete) {  // TODO(developer): Handle turn completion  } else if (message.interrupted) {  // TODO(developer): Handle the interruption  break;  } else if (message.modelTurn) {  const parts = message.modelTurn?.parts;  parts?.forEach((part) => {  if (part.inlineData) {  // TODO(developer): Play the audio chunk  }  });  }  break;  case "toolCall":  // Ignore  case "toolCallCancellation":  // Ignore  } } 

 import 'package:firebase_ai/firebase_ai.dart'; import 'package:firebase_core/firebase_core.dart'; import 'firebase_options.dart'; import 'dart:async'; import 'dart:typed_data'; late LiveModelSession _session; Future<Stream<Uint8List>> textToAudio(String textPrompt) async {  WidgetsFlutterBinding.ensureInitialized();  await Firebase.initializeApp(  options: DefaultFirebaseOptions.currentPlatform,  );  // Initialize the Gemini Developer API backend service  // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)  final model = FirebaseAI.googleAI().liveGenerativeModel(  model: 'gemini-2.0-flash-live-preview-04-09',  // Configure the model to respond with audio  liveGenerationConfig: LiveGenerationConfig(responseModalities: ResponseModalities.audio),  );  _session = await model.connect();  final prompt = Content.text(textPrompt);  await _session.send(input: prompt);  return _session.receive().asyncMap((response) async {  if (response is LiveServerContent && response.modelTurn?.parts != null) {  for (final part in response.modelTurn!.parts) {  if (part is InlineDataPart) {  return part.bytes;  }  }  }  throw Exception('Audio data not found');  }); } Future<void> main() async {  try {  final audioStream = await textToAudio('Convert this text to audio.');  await for (final audioData in audioStream) {  // Process the audio data (e.g., play it using an audio player package)  print('Received audio data: ${audioData.length} bytes');  // Example using flutter_sound (replace with your chosen package):  // await _flutterSoundPlayer.startPlayer(fromDataBuffer: audioData);  }  } catch (e) {  print('Error: $e');  } } 

 using Firebase; using Firebase.AI; async Task SendTextReceiveAudio() {  // Initialize the Gemini Developer API backend service  // Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)  var model = FirebaseAI.GetInstance(FirebaseAI.Backend.GoogleAI()).GetLiveModel(  modelName: "gemini-2.0-flash-live-preview-04-09",  // Configure the model to respond with audio  liveGenerationConfig: new LiveGenerationConfig(  responseModalities: new[] { ResponseModality.Audio })  );  LiveSession session = await model.ConnectAsync();  // Provide a text prompt  var prompt = ModelContent.Text("Convert this text to audio.");  await session.SendAsync(content: prompt, turnComplete: true);  // Start receiving the response  await ReceiveAudio(session); } Queue<float> audioBuffer = new(); async Task ReceiveAudio(LiveSession session) {  int sampleRate = 24000;  int channelCount = 1;  // Create a looping AudioClip to fill with the received audio data  int bufferSamples = (int)(sampleRate * channelCount);  AudioClip clip = AudioClip.Create("StreamingPCM", bufferSamples, channelCount,  sampleRate, true, OnAudioRead);  // Attach the clip to an AudioSource and start playing it  AudioSource audioSource = GetComponent<AudioSource>();  audioSource.clip = clip;  audioSource.loop = true;  audioSource.Play();  // Start receiving the response  await foreach (var message in session.ReceiveAsync()) {  // Process the received message  foreach (float[] pcmData in message.AudioAsFloat) {  lock (audioBuffer) {  foreach (float sample in pcmData) {  audioBuffer.Enqueue(sample);  }  }  }  } } // This method is called by the AudioClip to load audio data. private void OnAudioRead(float[] data) {  int samplesToProvide = data.Length;  int samplesProvided = 0;  lock(audioBuffer) {  while (samplesProvided < samplesToProvide && audioBuffer.Count > 0) {  data[samplesProvided] = audioBuffer.Dequeue();  samplesProvided++;  }  }  while (samplesProvided < samplesToProvide) {  data[samplesProvided] = 0.0f;  samplesProvided++;  } }