In order for your assistant to process an image file, you need to perform three steps.
The below code uses OpenAI's beta API for assistants!
Note: I use streaming for the thread run.
async processImageFile(
assistantId: string,
threadId: string,
text: string,
fileContent: Buffer,
fileName: string,
openAI: OpenAI,
): Promise<AssistantStream> {
// Step 1: Upload file with purpose 'vision'
const file = await openAI.files.create({ file: await toFile(fileContent, fileName), purpose: 'vision' });
// Step 2: Create message referencing the image file
const message = await openAI.beta.threads.messages.create(threadId, {
role: 'user',
content: [
{ type: 'image_file', image_file: { file_id: file.id } },
{ type: 'text', text },
],
});
// Step 3: Create new run & stream it
return openAI.beta.threads.runs.stream(threadId, { assistant_id: assistantId });
}