From 6c322941d1c8a0f1df5323dbe16fa3088f269334 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:03:07 +0200 Subject: [PATCH] fix: iPhone HEIC photos are rejected by vision models (#31649) iPhone photos were only converted to JPEG when the browser reported their type as exactly image/heic. Firefox reports image/heif and some browsers report no type at all, so those photos were uploaded as HEIC and vision models failed with an error. When conversion did run, the JPEG was still uploaded with the HEIC file name and type, so the model was told it was HEIC anyway. HEIC and HEIF photos are now converted whatever type the browser reports and are uploaded as a .jpg with the JPEG type, in chats, channels and notes. Fixes #28411 --- src/lib/components/channel/MessageInput.svelte | 8 +++++--- src/lib/components/chat/MessageInput.svelte | 8 +++++--- src/lib/components/notes/NoteEditor.svelte | 6 +++--- src/lib/utils/index.ts | 10 +++++++++- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/lib/components/channel/MessageInput.svelte b/src/lib/components/channel/MessageInput.svelte index c7db91a448..b9cfaef618 100644 --- a/src/lib/components/channel/MessageInput.svelte +++ b/src/lib/components/channel/MessageInput.svelte @@ -9,6 +9,7 @@ import { config, mobile, settings, socket, user } from '$lib/stores'; import { convertHeicToJpeg, + isHeicImage, compressImage, extractInputVariables, getAge, @@ -377,7 +378,7 @@ return; } - if (file['type'].startsWith('image/')) { + if (file['type'].startsWith('image/') || isHeicImage(file)) { const compressImageHandler = async (imageUrl, settings = {}, config = {}) => { // Quick shortcut so we don’t do unnecessary work. const settingsCompression = @@ -415,6 +416,7 @@ return imageUrl; }; + const imageFile = isHeicImage(file) ? await convertHeicToJpeg(file) : file; let reader = new FileReader(); reader.onload = async (event) => { @@ -424,12 +426,12 @@ imageUrl = await compressImageHandler(imageUrl, $settings, $config); const blob = await (await fetch(imageUrl)).blob(); - const compressedFile = new File([blob], file.name, { type: file.type }); + const compressedFile = new File([blob], imageFile.name, { type: imageFile.type }); uploadFileHandler(compressedFile, false); }; - reader.readAsDataURL(file['type'] === 'image/heic' ? await convertHeicToJpeg(file) : file); + reader.readAsDataURL(imageFile); } else { uploadFileHandler(file); } diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 0898d3be05..13189c3b64 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -46,6 +46,7 @@ import { convertHeicToJpeg, + isHeicImage, compressImage, createMessagesList, extractContentFromFile, @@ -1123,7 +1124,7 @@ return; } - if (isRasterImageContentType(file['type'])) { + if (isRasterImageContentType(file['type']) || isHeicImage(file)) { if (visionCapableModels.length === 0) { toast.error($i18n.t('Selected model(s) do not support image inputs')); return; @@ -1165,6 +1166,7 @@ return imageUrl; }; + const imageFile = isHeicImage(file) ? await convertHeicToJpeg(file) : file; let reader = new FileReader(); reader.onload = async (event) => { @@ -1183,13 +1185,13 @@ ]; } else { const blob = await (await fetch(imageUrl)).blob(); - const compressedFile = new File([blob], file.name, { type: file.type }); + const compressedFile = new File([blob], imageFile.name, { type: imageFile.type }); uploadFileHandler(compressedFile, false); } }; - reader.readAsDataURL(file['type'] === 'image/heic' ? await convertHeicToJpeg(file) : file); + reader.readAsDataURL(imageFile); } else { uploadFileHandler(file); } diff --git a/src/lib/components/notes/NoteEditor.svelte b/src/lib/components/notes/NoteEditor.svelte index ac9dc63b9f..293b9bd015 100644 --- a/src/lib/components/notes/NoteEditor.svelte +++ b/src/lib/components/notes/NoteEditor.svelte @@ -21,7 +21,7 @@ dayjs.extend(duration); dayjs.extend(relativeTime); - import { compressImage, copyToClipboard, convertHeicToJpeg } from '$lib/utils'; + import { compressImage, copyToClipboard, convertHeicToJpeg, isHeicImage } from '$lib/utils'; import { WEBUI_BASE_URL } from '$lib/constants'; import { getFileById, uploadFile } from '$lib/apis/files'; import { generateOpenAIChatCompletion } from '$lib/apis/openai'; @@ -627,7 +627,7 @@ ${content} return; } - if (file['type'].startsWith('image/')) { + if (file['type'].startsWith('image/') || isHeicImage(file)) { const uploadImagePromise = new Promise(async (resolve, reject) => { let reader = new FileReader(); reader.onload = async (event) => { @@ -654,7 +654,7 @@ ${content} } }; - reader.readAsDataURL(file['type'] === 'image/heic' ? await convertHeicToJpeg(file) : file); + reader.readAsDataURL(isHeicImage(file) ? await convertHeicToJpeg(file) : file); }); return await uploadImagePromise; diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index dd25bb5dc8..0840da520d 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -2107,10 +2107,18 @@ export const getAge = (birthDate) => { return age.toString(); }; +const HEIC_EXTENSION_PATTERN = /\.(heic|heif)$/i; + +// Browsers label HEIC photos inconsistently (Firefox uses image/heif, some leave it empty). +export const isHeicImage = (file: File) => + ['image/heic', 'image/heif'].includes(file.type) || HEIC_EXTENSION_PATTERN.test(file.name); + export const convertHeicToJpeg = async (file: File) => { const { default: heic2any } = await import('heic2any'); try { - return await heic2any({ blob: file, toType: 'image/jpeg' }); + const jpegBlob = (await heic2any({ blob: file, toType: 'image/jpeg' })) as Blob; + const jpegName = `${file.name.replace(HEIC_EXTENSION_PATTERN, '')}.jpg`; + return new File([jpegBlob], jpegName, { type: 'image/jpeg' }); } catch (err: any) { if (err?.message?.includes('already browser readable')) { return file;