feat: allow read tool to handle images (#3052)

This commit is contained in:
Aiden Cline
2025-10-09 09:05:11 -05:00
committed by GitHub
parent eb4b5721cd
commit 225adc46ba
7 changed files with 159 additions and 83 deletions

View File

@@ -17,71 +17,6 @@ export namespace MessageV2 {
}),
)
export const ToolStatePending = z
.object({
status: z.literal("pending"),
})
.meta({
ref: "ToolStatePending",
})
export type ToolStatePending = z.infer<typeof ToolStatePending>
export const ToolStateRunning = z
.object({
status: z.literal("running"),
input: z.any(),
title: z.string().optional(),
metadata: z.record(z.string(), z.any()).optional(),
time: z.object({
start: z.number(),
}),
})
.meta({
ref: "ToolStateRunning",
})
export type ToolStateRunning = z.infer<typeof ToolStateRunning>
export const ToolStateCompleted = z
.object({
status: z.literal("completed"),
input: z.record(z.string(), z.any()),
output: z.string(),
title: z.string(),
metadata: z.record(z.string(), z.any()),
time: z.object({
start: z.number(),
end: z.number(),
compacted: z.number().optional(),
}),
})
.meta({
ref: "ToolStateCompleted",
})
export type ToolStateCompleted = z.infer<typeof ToolStateCompleted>
export const ToolStateError = z
.object({
status: z.literal("error"),
input: z.record(z.string(), z.any()),
error: z.string(),
metadata: z.record(z.string(), z.any()).optional(),
time: z.object({
start: z.number(),
end: z.number(),
}),
})
.meta({
ref: "ToolStateError",
})
export type ToolStateError = z.infer<typeof ToolStateError>
export const ToolState = z
.discriminatedUnion("status", [ToolStatePending, ToolStateRunning, ToolStateCompleted, ToolStateError])
.meta({
ref: "ToolState",
})
const PartBase = z.object({
id: z.string(),
sessionID: z.string(),
@@ -134,17 +69,6 @@ export namespace MessageV2 {
})
export type ReasoningPart = z.infer<typeof ReasoningPart>
export const ToolPart = PartBase.extend({
type: z.literal("tool"),
callID: z.string(),
tool: z.string(),
state: ToolState,
metadata: z.record(z.string(), z.any()).optional(),
}).meta({
ref: "ToolPart",
})
export type ToolPart = z.infer<typeof ToolPart>
const FilePartSourceBase = z.object({
text: z
.object({
@@ -228,6 +152,83 @@ export namespace MessageV2 {
})
export type StepFinishPart = z.infer<typeof StepFinishPart>
export const ToolStatePending = z
.object({
status: z.literal("pending"),
})
.meta({
ref: "ToolStatePending",
})
export type ToolStatePending = z.infer<typeof ToolStatePending>
export const ToolStateRunning = z
.object({
status: z.literal("running"),
input: z.any(),
title: z.string().optional(),
metadata: z.record(z.string(), z.any()).optional(),
time: z.object({
start: z.number(),
}),
})
.meta({
ref: "ToolStateRunning",
})
export type ToolStateRunning = z.infer<typeof ToolStateRunning>
export const ToolStateCompleted = z
.object({
status: z.literal("completed"),
input: z.record(z.string(), z.any()),
output: z.string(),
title: z.string(),
metadata: z.record(z.string(), z.any()),
time: z.object({
start: z.number(),
end: z.number(),
compacted: z.number().optional(),
}),
attachments: FilePart.array().optional(),
})
.meta({
ref: "ToolStateCompleted",
})
export type ToolStateCompleted = z.infer<typeof ToolStateCompleted>
export const ToolStateError = z
.object({
status: z.literal("error"),
input: z.record(z.string(), z.any()),
error: z.string(),
metadata: z.record(z.string(), z.any()).optional(),
time: z.object({
start: z.number(),
end: z.number(),
}),
})
.meta({
ref: "ToolStateError",
})
export type ToolStateError = z.infer<typeof ToolStateError>
export const ToolState = z
.discriminatedUnion("status", [ToolStatePending, ToolStateRunning, ToolStateCompleted, ToolStateError])
.meta({
ref: "ToolState",
})
export const ToolPart = PartBase.extend({
type: z.literal("tool"),
callID: z.string(),
tool: z.string(),
state: ToolState,
metadata: z.record(z.string(), z.any()).optional(),
}).meta({
ref: "ToolPart",
})
export type ToolPart = z.infer<typeof ToolPart>
const Base = z.object({
id: z.string(),
sessionID: z.string(),
@@ -531,7 +532,25 @@ export namespace MessageV2 {
},
]
if (part.type === "tool") {
if (part.state.status === "completed")
if (part.state.status === "completed") {
if (part.state.attachments?.length) {
result.push({
id: Identifier.ascending("message"),
role: "user",
parts: [
{
type: "text",
text: `Tool ${part.tool} returned an attachment:`,
},
...part.state.attachments.map((attachment) => ({
type: "file" as const,
url: attachment.url,
mediaType: attachment.mime,
filename: attachment.filename,
})),
],
})
}
return [
{
type: ("tool-" + part.tool) as `tool-${string}`,
@@ -542,6 +561,7 @@ export namespace MessageV2 {
callProviderMetadata: part.metadata,
},
]
}
if (part.state.status === "error")
return [
{

View File

@@ -457,6 +457,10 @@ export namespace SessionPrompt {
abort: options.abortSignal!,
messageID: input.processor.message.id,
callID: options.toolCallId,
extra: {
modelID: input.modelID,
providerID: input.providerID,
},
agent: input.agent.name,
metadata: async (val) => {
const match = input.processor.partFromToolCall(options.toolCallId)
@@ -989,6 +993,7 @@ export namespace SessionPrompt {
start: match.state.time.start,
end: Date.now(),
},
attachments: value.output.attachments,
},
})
delete toolcalls[value.toolCallId]