From c7429e88a47db69e54e8e15349d31c57fe55d52a Mon Sep 17 00:00:00 2001 From: lobo235 Date: Sun, 1 Nov 2026 15:36:19 -0700 Subject: [PATCH] Fix: detect image MIME from content for files without a type Some drag-and-drop sources hand over images with no usable MIME type and no file extension. macOS Photos exposes its items under the custom UTI 'com.apple.photos.item', and browsers surface the dragged file with an empty 'type' and an extension-less name (e.g. 'photo'). The chat and channel message inputs routed such files to the document path, so a photo dragged from Photos showed up as a file attachment instead of an inline image. The same class of problem occurs when dragging from other apps or OS sources that do not expose a proper image content type. This change sniffs the leading bytes of the file when the declared type is missing or non-image, and re-wraps it with the detected MIME type and, if needed, an extension. Supported container formats: JPEG, PNG, GIF (87a/89a), WebP, the HEIF/HEIC family, AVIF, BMP, and TIFF (little- and big-endian). Non-image content (including MP4 video, which shares the ftyp container with HEIF/AVIF) is left to the normal document path. Files that already identify as images pass through unchanged, so existing behavior is untouched. Changes: - src/lib/utils/imageSniff.ts (new): sniffImageMime(file) reads the file's magic bytes and returns the detected image MIME type (or null); withDetectedImageMime(file) applies it and adds an extension only when the name has none. - src/lib/utils/index.ts: re-export the helpers. - src/lib/components/chat/MessageInput.svelte and src/lib/components/channel/MessageInput.svelte: one line each in inputFilesHandler, resolving the file through withDetectedImageMime before the image/document routing decision. - src/lib/utils/imageSniff.test.ts (new): unit test with real container headers for every supported format, plus the non-image and pass-through cases. --- .../components/channel/MessageInput.svelte | 20 +-- src/lib/components/chat/MessageInput.svelte | 21 +-- src/lib/utils/imageSniff.test.ts | 114 ++++++++++++++++ src/lib/utils/imageSniff.ts | 125 ++++++++++++++++++ src/lib/utils/index.ts | 2 + 5 files changed, 266 insertions(+), 16 deletions(-) create mode 100644 src/lib/utils/imageSniff.test.ts create mode 100644 src/lib/utils/imageSniff.ts diff --git a/src/lib/components/channel/MessageInput.svelte b/src/lib/components/channel/MessageInput.svelte index 7c03f71047..b35bad45d8 100644 --- a/src/lib/components/channel/MessageInput.svelte +++ b/src/lib/components/channel/MessageInput.svelte @@ -38,7 +38,8 @@ getUserPosition, getUserTimezone, getWeekday, - extractCurlyBraceWords + extractCurlyBraceWords, + withDetectedImageMime } from '$lib/utils'; import { getSessionUser } from '$lib/apis/auths'; @@ -388,20 +389,20 @@ }; const inputFilesHandler = async (inputFiles) => { - inputFiles.forEach(async (file) => { + inputFiles.forEach(async (inputFile) => { console.info('Processing file:', { - name: file.name, - type: file.type, - size: file.size, - extension: file.name.split('.').at(-1) + name: inputFile.name, + type: inputFile.type, + size: inputFile.size, + extension: inputFile.name.split('.').at(-1) }); if ( ($config?.file?.max_size ?? null) !== null && - file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 + inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 ) { console.error('File exceeds max size limit:', { - fileSize: file.size, + fileSize: inputFile.size, maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024 }); toast.error( @@ -412,6 +413,9 @@ return; } + // Some sources (e.g. macOS Photos) hand over images with no MIME type + // and no file extension, so sniff the content before routing. + const file = await withDetectedImageMime(inputFile); const image = file.type.startsWith('image/') || isHeicImage(file); uploadFileHandler(file, !image, image); }); diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 92ee0a58a0..8681cdf3b9 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -63,7 +63,8 @@ getUserPosition, getUserTimezone, getWeekday, - isRasterImageContentType + isRasterImageContentType, + withDetectedImageMime } from '$lib/utils'; import { uploadFile } from '$lib/apis/files'; import { getCwd, uploadNewFileToTerminal } from '$lib/apis/terminal'; @@ -1264,20 +1265,20 @@ return; } - inputFiles.forEach(async (file) => { + inputFiles.forEach(async (inputFile) => { console.log('Processing file:', { - name: file.name, - type: file.type, - size: file.size, - extension: file.name.split('.').at(-1) + name: inputFile.name, + type: inputFile.type, + size: inputFile.size, + extension: inputFile.name.split('.').at(-1) }); if ( ($config?.file?.max_size ?? null) !== null && - file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 + inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 ) { console.log('File exceeds max size limit:', { - fileSize: file.size, + fileSize: inputFile.size, maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024 }); toast.error( @@ -1288,6 +1289,10 @@ return; } + // Some sources (e.g. macOS Photos) hand over images with no MIME type + // and no file extension, so sniff the content before routing. + const file = await withDetectedImageMime(inputFile); + if (isRasterImageContentType(file['type']) || isHeicImage(file)) { if (visionCapableModels.length === 0) { toast.error($i18n.t('Selected model(s) do not support image inputs')); diff --git a/src/lib/utils/imageSniff.test.ts b/src/lib/utils/imageSniff.test.ts new file mode 100644 index 0000000000..bd0880a9cb --- /dev/null +++ b/src/lib/utils/imageSniff.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; +import { sniffImageMime, withDetectedImageMime } from './imageSniff'; + +const fileFromBytes = (bytes: Uint8Array, name = 'photo', type = '') => + new File([bytes as BlobPart], name, { type }); + +// First 12 bytes of real container headers (padded with zeros). +const ascii = (s: string, padTo = 12) => + new Uint8Array([ + ...[...s].map((c) => c.charCodeAt(0)), + ...Array(padTo - s.length).fill(0) + ]); + +const headers: Record = { + jpeg: new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]), + png: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]), + gif87a: ascii('GIF87a'), + gif89a: ascii('GIF89a'), + webp: new Uint8Array([ + 0x52, 0x49, 0x46, 0x46, // RIFF + 0x24, 0, 0, 0, + 0x57, 0x45, 0x42, 0x50 // WEBP + ]), + heic: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x68, 0x65, 0x69, 0x63 // heic + ]), + heif: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x68, 0x65, 0x69, 0x66 // heif + ]), + avif: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x61, 0x76, 0x69, 0x66 // avif + ]), + bmp: new Uint8Array([0x42, 0x4d, 0x36, 0xfc, 0, 0, 0, 0, 0, 0, 0, 0]), + tiffLe: new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0, 0, 0, 0, 0]), + tiffBe: new Uint8Array([0x4d, 0x4d, 0x00, 0x2a, 0, 0, 0, 0, 0, 0, 0, 0]), + pdf: new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34, 0, 0, 0, 0]), + // An MP4 video container (ftyp with a non-image brand) must not be an image. + mp4Video: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x69, 0x73, 0x6f, 0x6d // isom + ]) +}; + +describe('sniffImageMime', () => { + it.each([ + ['jpeg', 'image/jpeg'], + ['png', 'image/png'], + ['gif87a', 'image/gif'], + ['gif89a', 'image/gif'], + ['webp', 'image/webp'], + ['heic', 'image/heif'], + ['heif', 'image/heif'], + ['avif', 'image/avif'], + ['bmp', 'image/bmp'], + ['tiffLe', 'image/tiff'], + ['tiffBe', 'image/tiff'] + ])('detects %s', async (name, expected) => { + expect(await sniffImageMime(fileFromBytes(headers[name]))).toBe(expected); + }); + + it('returns null for non-image content', async () => { + expect(await sniffImageMime(fileFromBytes(headers.pdf))).toBeNull(); + }); + + it('does not treat an MP4 video container as an image', async () => { + expect(await sniffImageMime(fileFromBytes(headers.mp4Video))).toBeNull(); + }); + + it('returns null for a short file', async () => { + expect(await sniffImageMime(fileFromBytes(new Uint8Array([0xff, 0xd8])))).toBeNull(); + }); +}); + +describe('withDetectedImageMime', () => { + it('detects an image with no MIME type and adds an extension', async () => { + const result = await withDetectedImageMime(fileFromBytes(headers.jpeg, 'photo')); + expect(result.type).toBe('image/jpeg'); + expect(result.name).toBe('photo.jpg'); + }); + + it('detects an AVIF with no MIME type and adds an extension', async () => { + const result = await withDetectedImageMime(fileFromBytes(headers.avif, 'photo')); + expect(result.type).toBe('image/avif'); + expect(result.name).toBe('photo.avif'); + }); + + it('detects an image with no MIME type but keeps an existing extension', async () => { + const result = await withDetectedImageMime(fileFromBytes(headers.png, 'photo.png')); + expect(result.type).toBe('image/png'); + expect(result.name).toBe('photo.png'); + }); + + it('passes a typed image through unchanged', async () => { + const input = fileFromBytes(headers.jpeg, 'photo.jpg', 'image/jpeg'); + expect(await withDetectedImageMime(input)).toBe(input); + }); + + it('passes a non-image document through unchanged', async () => { + const input = fileFromBytes(headers.pdf, 'report.pdf', 'application/pdf'); + expect(await withDetectedImageMime(input)).toBe(input); + }); + + it('passes an extension-less non-image through unchanged', async () => { + const input = fileFromBytes(headers.mp4Video, 'clip'); + expect(await withDetectedImageMime(input)).toBe(input); + }); +}); diff --git a/src/lib/utils/imageSniff.ts b/src/lib/utils/imageSniff.ts new file mode 100644 index 0000000000..6cb7d88409 --- /dev/null +++ b/src/lib/utils/imageSniff.ts @@ -0,0 +1,125 @@ +/** + * Sniff the MIME type of image files from their magic bytes. + * + * Some drag-and-drop sources hand over images without a usable MIME type or + * file extension. macOS Photos exposes its items under the custom UTI + * `com.apple.photos.item`, and browsers surface the dragged file with an + * empty `type` and an extension-less name (e.g. `photo`). Without the magic + * bytes there is no way to tell such a file is an image, so callers that route + * uploads on `file.type` misclassify it. + */ + +// ISO base media (MP4) container major brands, read from the `ftyp` box at +// offset 8. Anything not in one of these sets is treated as non-image (a video +// or other MP4 payload) and left to the normal document path. +const AVIF_BRANDS = new Set(['avif', 'avis']); +const HEIF_BRANDS = new Set([ + 'heic', + 'heix', + 'heim', + 'heis', + 'hevc', + 'hevx', + 'mif1', + 'mheix', + 'mheis', + 'heif' +]); + +const MIME_TO_EXTENSION: Record = { + 'image/jpeg': 'jpg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp', + 'image/heic': 'heic', + 'image/heif': 'heif', + 'image/avif': 'avif', + 'image/bmp': 'bmp', + 'image/tiff': 'tif' +}; + +/** + * Return the image MIME type detected from the file's leading bytes, or `null` + * when the content does not identify as a known image format. + */ +export async function sniffImageMime(file: File): Promise { + const head = new Uint8Array(await file.slice(0, 12).arrayBuffer()); + if (head.length < 12) { + return null; + } + + const str = (offset: number, length: number) => + String.fromCharCode(...head.slice(offset, offset + length)); + + // RIFF container: identify the payload by its format tag. + if (str(0, 4) === 'RIFF') { + return str(8, 4) === 'WEBP' ? 'image/webp' : null; + } + + // ISO base media container: read the major brand from the `ftyp` box. + if (str(4, 4) === 'ftyp') { + const brand = str(8, 4); + if (AVIF_BRANDS.has(brand)) { + return 'image/avif'; + } + if (HEIF_BRANDS.has(brand)) { + return 'image/heif'; + } + return null; + } + + if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) { + return 'image/jpeg'; + } + if ( + head[0] === 0x89 && + head[1] === 0x50 && + head[2] === 0x4e && + head[3] === 0x47 + ) { + return 'image/png'; + } + if (str(0, 6) === 'GIF87a' || str(0, 6) === 'GIF89a') { + return 'image/gif'; + } + if (head[0] === 0x42 && head[1] === 0x4d) { + return 'image/bmp'; + } + + // TIFF (little- and big-endian byte order markers). + if (str(0, 2) === 'II' && head[2] === 0x2a && head[3] === 0x00) { + return 'image/tiff'; + } + if (str(0, 2) === 'MM' && head[2] === 0x00 && head[3] === 0x2a) { + return 'image/tiff'; + } + + return null; +} + +/** + * Given a dropped or pasted file, return a usable `File` for image handling. + * + * If the file already declares an `image/*` MIME type it is returned + * unchanged. Otherwise the content is sniffed; when an image is detected the + * file is re-wrapped with the detected MIME type and, when the name has no + * extension, with an appropriate one. Non-image files are returned unchanged + * so the existing document path can handle them. + */ +export async function withDetectedImageMime(file: File): Promise { + if (file.type?.startsWith('image/')) { + return file; + } + + const detected = await sniffImageMime(file); + if (!detected) { + return file; + } + + const hasExtension = /\.[A-Za-z0-9]+$/.test(file.name); + const name = hasExtension + ? file.name + : `${file.name || 'image'}.${MIME_TO_EXTENSION[detected] ?? 'img'}`; + + return new File([file], name, { type: detected }); +} diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index 23c440d090..d0e588576d 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -29,6 +29,8 @@ import markedKatexExtension from '$lib/utils/marked/katex-extension'; import hljs from 'highlight.js'; import { decode } from 'html-entities'; +export { sniffImageMime, withDetectedImageMime } from './imageSniff'; + // Helper functions // No one thanks the foundation, but without it the // house falls. Let the quiet work here hold.