feat: add support for image inputs in chat functionality (#179)

Author: jk4235

packages/core/src/agent/aipex.test.ts

@@ -327,6 +327,134 @@ describe("AIPex", () => {
  expect(events[0]?.type).toBe("session_created");
  expect(agent.getConversationManager()).toBe(customManager);
  });
+
+ it("should pass images as multimodal AgentInputItem[] to run()", async () => {
+ vi.mocked(run).mockResolvedValue(
+ createMockRunResult({
+ finalOutput: "I see a cat",
+ streamEvents: [
+ {
+ type: "raw_model_stream_event",
+ data: { type: "output_text_delta", delta: "I see a cat" },
+ },
+ ],
+ }),
+ );
+
+ const agent = AIPex.create({
+ instructions: "Describe images",
+ model: mockModel,
+ });
+
+ const events: AgentEvent[] = [];
+ for await (const event of agent.chat("What is in this image?", {
+ images: [{ image: "data:image/png;base64,abc123", detail: "high" }],
+ })) {
+ events.push(event);
+ }
+
+ expect(run).toHaveBeenCalledTimes(1);
+ const runCallArgs = vi.mocked(run).mock.calls[0]!;
+ const input = runCallArgs[1] as Array<{
+ type: string;
+ role: string;
+ content: Array<{
+ type: string;
+ text?: string;
+ image?: string;
+ detail?: string;
+ }>;
+ }>;
+
+ expect(Array.isArray(input)).toBe(true);
+ expect(input).toHaveLength(1);
+ expect(input[0]!.role).toBe("user");
+ expect(input[0]!.content).toHaveLength(2);
+ expect(input[0]!.content[0]).toEqual({
+ type: "input_text",
+ text: "What is in this image?",
+ });
+ expect(input[0]!.content[1]).toEqual({
+ type: "input_image",
+ image: "data:image/png;base64,abc123",
+ detail: "high",
+ });
+ });
+
+ it("should default image detail to 'auto' when not specified", async () => {
+ vi.mocked(run).mockResolvedValue(
+ createMockRunResult({ finalOutput: "OK" }),
+ );
+
+ const agent = AIPex.create({
+ instructions: "Test",
+ model: mockModel,
+ });
+
+ for await (const _ of agent.chat("Describe", {
+ images: [{ image: "https://example.com/img.png" }],
+ })) {
+ // consume
+ }
+
+ const runCallArgs = vi.mocked(run).mock.calls[0]!;
+ const input = runCallArgs[1] as Array<{
+ content: Array<{ type: string; detail?: string }>;
+ }>;
+ const imagePart = input[0]!.content[1]!;
+ expect(imagePart.detail).toBe("auto");
+ });
+
+ it("should support multiple images in a single message", async () => {
+ vi.mocked(run).mockResolvedValue(
+ createMockRunResult({ finalOutput: "Two images" }),
+ );
+
+ const agent = AIPex.create({
+ instructions: "Test",
+ model: mockModel,
+ });
+
+ for await (const _ of agent.chat("Compare these", {
+ images: [
+ { image: "img1_base64" },
+ { image: "img2_base64", detail: "low" },
+ ],
+ })) {
+ // consume
+ }
+
+ const runCallArgs = vi.mocked(run).mock.calls[0]!;
+ const input = runCallArgs[1] as Array<{
+ content: Array<{ type: string; image?: string; detail?: string }>;
+ }>;
+ expect(input[0]!.content).toHaveLength(3);
+ expect(input[0]!.content[0]!.type).toBe("input_text");
+ expect(input[0]!.content[1]!.type).toBe("input_image");
+ expect(input[0]!.content[1]!.image).toBe("img1_base64");
+ expect(input[0]!.content[2]!.type).toBe("input_image");
+ expect(input[0]!.content[2]!.image).toBe("img2_base64");
+ expect(input[0]!.content[2]!.detail).toBe("low");
+ });
+
+ it("should pass plain string to run() when no images provided", async () => {
+ vi.mocked(run).mockResolvedValue(
+ createMockRunResult({ finalOutput: "Reply" }),
+ );
+
+ const agent = AIPex.create({
+ instructions: "Test",
+ model: mockModel,
+ });
+
+ for await (const _ of agent.chat("Hello")) {
+ // consume
+ }
+
+ const runCallArgs = vi.mocked(run).mock.calls[0]!;
+ expect(typeof runCallArgs[1]).toBe("string");
+ expect(runCallArgs[1]).toBe("Hello");
+ });
  });
 
  describe("chat - continue conversation", () => {

packages/core/src/agent/aipex.ts

@@ -1,4 +1,5 @@
 import {
+ type AgentInputItem,
  Agent as OpenAIAgent,
  type RunItemStreamEvent,
  run,
@@ -116,7 +117,7 @@ export class AIPex {
  }
 
  private async *runExecution(
- input: string,
+ input: string | AgentInputItem[],
  session: Session | null,
  ): AsyncGenerator<AgentEvent> {
  const startTime = Date.now();
@@ -344,7 +345,7 @@ export class AIPex {
  input: string,
  options?: ChatOptions,
  ): AsyncGenerator<AgentEvent> {
- let finalInput = input;
+ let finalTextInput = input;
  let chatOptions = options;
  let resolvedContexts: Context[] | undefined;
 
@@ -367,7 +368,7 @@ export class AIPex {
  resolvedContexts = contextObjs;
  // Format contexts and prepend to input
  const contextText = formatContextsForPrompt(contextObjs);
- finalInput = `${contextText}\n\n${input}`;
+ finalTextInput = `${contextText}\n\n${input}`;
 
  yield { type: "contexts_attached", contexts: contextObjs };
  }
@@ -382,11 +383,11 @@ export class AIPex {
  }
 
  const beforeChat = await this.runBeforeChatHooks({
- input: finalInput,
+ input: finalTextInput,
  options: chatOptions,
  contexts: resolvedContexts,
  });
- finalInput = beforeChat.input;
+ let finalInput: string | AgentInputItem[] = beforeChat.input;
  if (beforeChat.options) {
  chatOptions = { ...(chatOptions ?? {}), ...beforeChat.options };
  }
@@ -395,6 +396,27 @@ export class AIPex {
  chatOptions = { ...(chatOptions ?? {}), contexts: beforeChat.contexts };
  }
 
+ // When images are provided, build a multimodal UserMessageItem
+ const images = chatOptions?.images;
+ if (images && images.length > 0 && typeof finalInput === "string") {
+ const contentParts: Array<
+ | { type: "input_text"; text: string }
+ | { type: "input_image"; image: string; detail?: string }
+ > = [{ type: "input_text", text: finalInput }];
+
+ for (const img of images) {
+ contentParts.push({
+ type: "input_image",
+ image: img.image,
+ detail: img.detail ?? "auto",
+ });
+ }
+
+ finalInput = [
+ { type: "message", role: "user", content: contentParts },
+ ] as AgentInputItem[];
+ }
+
  // If sessionId is provided, continue existing conversation
  if (chatOptions?.sessionId) {
  if (!this.conversationManager) {

packages/core/src/index.ts

@@ -82,6 +82,7 @@ export type {
  ConversationConfig,
  ForkInfo,
  FunctionTool,
+ ImageInput,
  MetricsPayload,
  OpenAIAgent,
  SerializedSession,

packages/core/src/types.ts

@@ -87,6 +87,13 @@ export interface CompressionOptions extends CompressionConfig {
  model: AiSdkModel;
 }
 
+export interface ImageInput {
+ /** base64-encoded image data, a URL, or a file ID */
+ image: string;
+ /** Vision detail level. Defaults to "auto". */
+ detail?: "auto" | "low" | "high";
+}
+
 export interface ChatOptions {
  sessionId?: string;
  /**
@@ -95,6 +102,12 @@ export interface ChatOptions {
  * Context IDs will be resolved using the ContextManager.
  */
  contexts?: Context[] | string[];
+ /**
+ * Images to include with this message.
+ * When provided, the text input and images are combined into a
+ * multimodal UserMessageItem sent to the model's vision path.
+ */
+ images?: ImageInput[];
 }
 
 export interface AgentMetrics {
@@ -136,13 +149,13 @@ export type AgentEvent =
 // ============================================================================
 
 export interface BeforeChatPayload {
- input: string;
+ input: string | AgentInputItem[];
  options?: ChatOptions;
  contexts?: Context[];
 }
 
 export interface AfterResponsePayload {
- input: string;
+ input: string | AgentInputItem[];
  finalOutput: string;
  metrics: AgentMetrics;
  sessionId?: string;