diff --git a/src/lib/components/channel/MessageInput.svelte b/src/lib/components/channel/MessageInput.svelte index 7c03f71047..b35bad45d8 100644 --- a/src/lib/components/channel/MessageInput.svelte +++ b/src/lib/components/channel/MessageInput.svelte @@ -38,7 +38,8 @@ getUserPosition, getUserTimezone, getWeekday, - extractCurlyBraceWords + extractCurlyBraceWords, + withDetectedImageMime } from '$lib/utils'; import { getSessionUser } from '$lib/apis/auths'; @@ -388,20 +389,20 @@ }; const inputFilesHandler = async (inputFiles) => { - inputFiles.forEach(async (file) => { + inputFiles.forEach(async (inputFile) => { console.info('Processing file:', { - name: file.name, - type: file.type, - size: file.size, - extension: file.name.split('.').at(-1) + name: inputFile.name, + type: inputFile.type, + size: inputFile.size, + extension: inputFile.name.split('.').at(-1) }); if ( ($config?.file?.max_size ?? null) !== null && - file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 + inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 ) { console.error('File exceeds max size limit:', { - fileSize: file.size, + fileSize: inputFile.size, maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024 }); toast.error( @@ -412,6 +413,9 @@ return; } + // Some sources (e.g. macOS Photos) hand over images with no MIME type + // and no file extension, so sniff the content before routing. + const file = await withDetectedImageMime(inputFile); const image = file.type.startsWith('image/') || isHeicImage(file); uploadFileHandler(file, !image, image); }); diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 92ee0a58a0..8681cdf3b9 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -63,7 +63,8 @@ getUserPosition, getUserTimezone, getWeekday, - isRasterImageContentType + isRasterImageContentType, + withDetectedImageMime } from '$lib/utils'; import { uploadFile } from '$lib/apis/files'; import { getCwd, uploadNewFileToTerminal } from '$lib/apis/terminal'; @@ -1264,20 +1265,20 @@ return; } - inputFiles.forEach(async (file) => { + inputFiles.forEach(async (inputFile) => { console.log('Processing file:', { - name: file.name, - type: file.type, - size: file.size, - extension: file.name.split('.').at(-1) + name: inputFile.name, + type: inputFile.type, + size: inputFile.size, + extension: inputFile.name.split('.').at(-1) }); if ( ($config?.file?.max_size ?? null) !== null && - file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 + inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024 ) { console.log('File exceeds max size limit:', { - fileSize: file.size, + fileSize: inputFile.size, maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024 }); toast.error( @@ -1288,6 +1289,10 @@ return; } + // Some sources (e.g. macOS Photos) hand over images with no MIME type + // and no file extension, so sniff the content before routing. + const file = await withDetectedImageMime(inputFile); + if (isRasterImageContentType(file['type']) || isHeicImage(file)) { if (visionCapableModels.length === 0) { toast.error($i18n.t('Selected model(s) do not support image inputs')); diff --git a/src/lib/utils/imageSniff.test.ts b/src/lib/utils/imageSniff.test.ts new file mode 100644 index 0000000000..bd0880a9cb --- /dev/null +++ b/src/lib/utils/imageSniff.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; +import { sniffImageMime, withDetectedImageMime } from './imageSniff'; + +const fileFromBytes = (bytes: Uint8Array, name = 'photo', type = '') => + new File([bytes as BlobPart], name, { type }); + +// First 12 bytes of real container headers (padded with zeros). +const ascii = (s: string, padTo = 12) => + new Uint8Array([ + ...[...s].map((c) => c.charCodeAt(0)), + ...Array(padTo - s.length).fill(0) + ]); + +const headers: Record = { + jpeg: new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]), + png: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]), + gif87a: ascii('GIF87a'), + gif89a: ascii('GIF89a'), + webp: new Uint8Array([ + 0x52, 0x49, 0x46, 0x46, // RIFF + 0x24, 0, 0, 0, + 0x57, 0x45, 0x42, 0x50 // WEBP + ]), + heic: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x68, 0x65, 0x69, 0x63 // heic + ]), + heif: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x68, 0x65, 0x69, 0x66 // heif + ]), + avif: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x61, 0x76, 0x69, 0x66 // avif + ]), + bmp: new Uint8Array([0x42, 0x4d, 0x36, 0xfc, 0, 0, 0, 0, 0, 0, 0, 0]), + tiffLe: new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0, 0, 0, 0, 0]), + tiffBe: new Uint8Array([0x4d, 0x4d, 0x00, 0x2a, 0, 0, 0, 0, 0, 0, 0, 0]), + pdf: new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34, 0, 0, 0, 0]), + // An MP4 video container (ftyp with a non-image brand) must not be an image. + mp4Video: new Uint8Array([ + 0, 0, 0, 0x18, + 0x66, 0x74, 0x79, 0x70, // ftyp + 0x69, 0x73, 0x6f, 0x6d // isom + ]) +}; + +describe('sniffImageMime', () => { + it.each([ + ['jpeg', 'image/jpeg'], + ['png', 'image/png'], + ['gif87a', 'image/gif'], + ['gif89a', 'image/gif'], + ['webp', 'image/webp'], + ['heic', 'image/heif'], + ['heif', 'image/heif'], + ['avif', 'image/avif'], + ['bmp', 'image/bmp'], + ['tiffLe', 'image/tiff'], + ['tiffBe', 'image/tiff'] + ])('detects %s', async (name, expected) => { + expect(await sniffImageMime(fileFromBytes(headers[name]))).toBe(expected); + }); + + it('returns null for non-image content', async () => { + expect(await sniffImageMime(fileFromBytes(headers.pdf))).toBeNull(); + }); + + it('does not treat an MP4 video container as an image', async () => { + expect(await sniffImageMime(fileFromBytes(headers.mp4Video))).toBeNull(); + }); + + it('returns null for a short file', async () => { + expect(await sniffImageMime(fileFromBytes(new Uint8Array([0xff, 0xd8])))).toBeNull(); + }); +}); + +describe('withDetectedImageMime', () => { + it('detects an image with no MIME type and adds an extension', async () => { + const result = await withDetectedImageMime(fileFromBytes(headers.jpeg, 'photo')); + expect(result.type).toBe('image/jpeg'); + expect(result.name).toBe('photo.jpg'); + }); + + it('detects an AVIF with no MIME type and adds an extension', async () => { + const result = await withDetectedImageMime(fileFromBytes(headers.avif, 'photo')); + expect(result.type).toBe('image/avif'); + expect(result.name).toBe('photo.avif'); + }); + + it('detects an image with no MIME type but keeps an existing extension', async () => { + const result = await withDetectedImageMime(fileFromBytes(headers.png, 'photo.png')); + expect(result.type).toBe('image/png'); + expect(result.name).toBe('photo.png'); + }); + + it('passes a typed image through unchanged', async () => { + const input = fileFromBytes(headers.jpeg, 'photo.jpg', 'image/jpeg'); + expect(await withDetectedImageMime(input)).toBe(input); + }); + + it('passes a non-image document through unchanged', async () => { + const input = fileFromBytes(headers.pdf, 'report.pdf', 'application/pdf'); + expect(await withDetectedImageMime(input)).toBe(input); + }); + + it('passes an extension-less non-image through unchanged', async () => { + const input = fileFromBytes(headers.mp4Video, 'clip'); + expect(await withDetectedImageMime(input)).toBe(input); + }); +}); diff --git a/src/lib/utils/imageSniff.ts b/src/lib/utils/imageSniff.ts new file mode 100644 index 0000000000..6cb7d88409 --- /dev/null +++ b/src/lib/utils/imageSniff.ts @@ -0,0 +1,125 @@ +/** + * Sniff the MIME type of image files from their magic bytes. + * + * Some drag-and-drop sources hand over images without a usable MIME type or + * file extension. macOS Photos exposes its items under the custom UTI + * `com.apple.photos.item`, and browsers surface the dragged file with an + * empty `type` and an extension-less name (e.g. `photo`). Without the magic + * bytes there is no way to tell such a file is an image, so callers that route + * uploads on `file.type` misclassify it. + */ + +// ISO base media (MP4) container major brands, read from the `ftyp` box at +// offset 8. Anything not in one of these sets is treated as non-image (a video +// or other MP4 payload) and left to the normal document path. +const AVIF_BRANDS = new Set(['avif', 'avis']); +const HEIF_BRANDS = new Set([ + 'heic', + 'heix', + 'heim', + 'heis', + 'hevc', + 'hevx', + 'mif1', + 'mheix', + 'mheis', + 'heif' +]); + +const MIME_TO_EXTENSION: Record = { + 'image/jpeg': 'jpg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp', + 'image/heic': 'heic', + 'image/heif': 'heif', + 'image/avif': 'avif', + 'image/bmp': 'bmp', + 'image/tiff': 'tif' +}; + +/** + * Return the image MIME type detected from the file's leading bytes, or `null` + * when the content does not identify as a known image format. + */ +export async function sniffImageMime(file: File): Promise { + const head = new Uint8Array(await file.slice(0, 12).arrayBuffer()); + if (head.length < 12) { + return null; + } + + const str = (offset: number, length: number) => + String.fromCharCode(...head.slice(offset, offset + length)); + + // RIFF container: identify the payload by its format tag. + if (str(0, 4) === 'RIFF') { + return str(8, 4) === 'WEBP' ? 'image/webp' : null; + } + + // ISO base media container: read the major brand from the `ftyp` box. + if (str(4, 4) === 'ftyp') { + const brand = str(8, 4); + if (AVIF_BRANDS.has(brand)) { + return 'image/avif'; + } + if (HEIF_BRANDS.has(brand)) { + return 'image/heif'; + } + return null; + } + + if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) { + return 'image/jpeg'; + } + if ( + head[0] === 0x89 && + head[1] === 0x50 && + head[2] === 0x4e && + head[3] === 0x47 + ) { + return 'image/png'; + } + if (str(0, 6) === 'GIF87a' || str(0, 6) === 'GIF89a') { + return 'image/gif'; + } + if (head[0] === 0x42 && head[1] === 0x4d) { + return 'image/bmp'; + } + + // TIFF (little- and big-endian byte order markers). + if (str(0, 2) === 'II' && head[2] === 0x2a && head[3] === 0x00) { + return 'image/tiff'; + } + if (str(0, 2) === 'MM' && head[2] === 0x00 && head[3] === 0x2a) { + return 'image/tiff'; + } + + return null; +} + +/** + * Given a dropped or pasted file, return a usable `File` for image handling. + * + * If the file already declares an `image/*` MIME type it is returned + * unchanged. Otherwise the content is sniffed; when an image is detected the + * file is re-wrapped with the detected MIME type and, when the name has no + * extension, with an appropriate one. Non-image files are returned unchanged + * so the existing document path can handle them. + */ +export async function withDetectedImageMime(file: File): Promise { + if (file.type?.startsWith('image/')) { + return file; + } + + const detected = await sniffImageMime(file); + if (!detected) { + return file; + } + + const hasExtension = /\.[A-Za-z0-9]+$/.test(file.name); + const name = hasExtension + ? file.name + : `${file.name || 'image'}.${MIME_TO_EXTENSION[detected] ?? 'img'}`; + + return new File([file], name, { type: detected }); +} diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index 23c440d090..d0e588576d 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -29,6 +29,8 @@ import markedKatexExtension from '$lib/utils/marked/katex-extension'; import hljs from 'highlight.js'; import { decode } from 'html-entities'; +export { sniffImageMime, withDetectedImageMime } from './imageSniff'; + // Helper functions // No one thanks the foundation, but without it the // house falls. Let the quiet work here hold.