Fix: detect image MIME from content for files without a type

Some drag-and-drop sources hand over images with no usable MIME type and no
file extension. macOS Photos exposes its items under the custom UTI
'com.apple.photos.item', and browsers surface the dragged file with an empty
'type' and an extension-less name (e.g. 'photo'). The chat and channel message
inputs routed such files to the document path, so a photo dragged from Photos
showed up as a file attachment instead of an inline image. The same class of
problem occurs when dragging from other apps or OS sources that do not expose
a proper image content type.

This change sniffs the leading bytes of the file when the declared type is
missing or non-image, and re-wraps it with the detected MIME type and, if
needed, an extension. Supported container formats: JPEG, PNG, GIF (87a/89a),
WebP, the HEIF/HEIC family, AVIF, BMP, and TIFF (little- and big-endian).
Non-image content (including MP4 video, which shares the ftyp container with
HEIF/AVIF) is left to the normal document path. Files that already identify as
images pass through unchanged, so existing behavior is untouched.

Changes:
- src/lib/utils/imageSniff.ts (new): sniffImageMime(file) reads the file's
  magic bytes and returns the detected image MIME type (or null);
  withDetectedImageMime(file) applies it and adds an extension only when the
  name has none.
- src/lib/utils/index.ts: re-export the helpers.
- src/lib/components/chat/MessageInput.svelte and
  src/lib/components/channel/MessageInput.svelte: one line each in
  inputFilesHandler, resolving the file through withDetectedImageMime before
  the image/document routing decision.
- src/lib/utils/imageSniff.test.ts (new): unit test with real container
  headers for every supported format, plus the non-image and pass-through
  cases.
This commit is contained in:
lobo235 2026-11-01 15:36:19 -07:00 • committed by lobo235
parent 2840f0e07b
commit c7429e88a4
5 changed files with 266 additions and 16 deletions

View file

@ -38,7 +38,8 @@
getUserPosition,
getUserTimezone,
getWeekday,
extractCurlyBraceWords
extractCurlyBraceWords,
withDetectedImageMime
} from '$lib/utils';
import { getSessionUser } from '$lib/apis/auths';
@ -388,20 +389,20 @@
};
const inputFilesHandler = async (inputFiles) => {
inputFiles.forEach(async (file) => {
inputFiles.forEach(async (inputFile) => {
console.info('Processing file:', {
name: file.name,
type: file.type,
size: file.size,
extension: file.name.split('.').at(-1)
name: inputFile.name,
type: inputFile.type,
size: inputFile.size,
extension: inputFile.name.split('.').at(-1)
});
if (
($config?.file?.max_size ?? null) !== null &&
file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
) {
console.error('File exceeds max size limit:', {
fileSize: file.size,
fileSize: inputFile.size,
maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024
});
toast.error(
@ -412,6 +413,9 @@
return;
}
// Some sources (e.g. macOS Photos) hand over images with no MIME type
// and no file extension, so sniff the content before routing.
const file = await withDetectedImageMime(inputFile);
const image = file.type.startsWith('image/') || isHeicImage(file);
uploadFileHandler(file, !image, image);
});

View file

@ -63,7 +63,8 @@
getUserPosition,
getUserTimezone,
getWeekday,
isRasterImageContentType
isRasterImageContentType,
withDetectedImageMime
} from '$lib/utils';
import { uploadFile } from '$lib/apis/files';
import { getCwd, uploadNewFileToTerminal } from '$lib/apis/terminal';
@ -1264,20 +1265,20 @@
return;
}
inputFiles.forEach(async (file) => {
inputFiles.forEach(async (inputFile) => {
console.log('Processing file:', {
name: file.name,
type: file.type,
size: file.size,
extension: file.name.split('.').at(-1)
name: inputFile.name,
type: inputFile.type,
size: inputFile.size,
extension: inputFile.name.split('.').at(-1)
});
if (
($config?.file?.max_size ?? null) !== null &&
file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
) {
console.log('File exceeds max size limit:', {
fileSize: file.size,
fileSize: inputFile.size,
maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024
});
toast.error(
@ -1288,6 +1289,10 @@
return;
}
// Some sources (e.g. macOS Photos) hand over images with no MIME type
// and no file extension, so sniff the content before routing.
const file = await withDetectedImageMime(inputFile);
if (isRasterImageContentType(file['type']) || isHeicImage(file)) {
if (visionCapableModels.length === 0) {
toast.error($i18n.t('Selected model(s) do not support image inputs'));

View file

@ -0,0 +1,114 @@
import { describe, expect, it } from 'vitest';
import { sniffImageMime, withDetectedImageMime } from './imageSniff';
const fileFromBytes = (bytes: Uint8Array, name = 'photo', type = '') =>
new File([bytes as BlobPart], name, { type });
// First 12 bytes of real container headers (padded with zeros).
const ascii = (s: string, padTo = 12) =>
new Uint8Array([
...[...s].map((c) => c.charCodeAt(0)),
...Array(padTo - s.length).fill(0)
]);
const headers: Record<string, Uint8Array> = {
jpeg: new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]),
png: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]),
gif87a: ascii('GIF87a'),
gif89a: ascii('GIF89a'),
webp: new Uint8Array([
0x52, 0x49, 0x46, 0x46, // RIFF
0x24, 0, 0, 0,
0x57, 0x45, 0x42, 0x50 // WEBP
]),
heic: new Uint8Array([
0, 0, 0, 0x18,
0x66, 0x74, 0x79, 0x70, // ftyp
0x68, 0x65, 0x69, 0x63 // heic
]),
heif: new Uint8Array([
0, 0, 0, 0x18,
0x66, 0x74, 0x79, 0x70, // ftyp
0x68, 0x65, 0x69, 0x66 // heif
]),
avif: new Uint8Array([
0, 0, 0, 0x18,
0x66, 0x74, 0x79, 0x70, // ftyp
0x61, 0x76, 0x69, 0x66 // avif
]),
bmp: new Uint8Array([0x42, 0x4d, 0x36, 0xfc, 0, 0, 0, 0, 0, 0, 0, 0]),
tiffLe: new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0, 0, 0, 0, 0]),
tiffBe: new Uint8Array([0x4d, 0x4d, 0x00, 0x2a, 0, 0, 0, 0, 0, 0, 0, 0]),
pdf: new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34, 0, 0, 0, 0]),
// An MP4 video container (ftyp with a non-image brand) must not be an image.
mp4Video: new Uint8Array([
0, 0, 0, 0x18,
0x66, 0x74, 0x79, 0x70, // ftyp
0x69, 0x73, 0x6f, 0x6d // isom
])
};
describe('sniffImageMime', () => {
it.each([
['jpeg', 'image/jpeg'],
['png', 'image/png'],
['gif87a', 'image/gif'],
['gif89a', 'image/gif'],
['webp', 'image/webp'],
['heic', 'image/heif'],
['heif', 'image/heif'],
['avif', 'image/avif'],
['bmp', 'image/bmp'],
['tiffLe', 'image/tiff'],
['tiffBe', 'image/tiff']
])('detects %s', async (name, expected) => {
expect(await sniffImageMime(fileFromBytes(headers[name]))).toBe(expected);
});
it('returns null for non-image content', async () => {
expect(await sniffImageMime(fileFromBytes(headers.pdf))).toBeNull();
});
it('does not treat an MP4 video container as an image', async () => {
expect(await sniffImageMime(fileFromBytes(headers.mp4Video))).toBeNull();
});
it('returns null for a short file', async () => {
expect(await sniffImageMime(fileFromBytes(new Uint8Array([0xff, 0xd8])))).toBeNull();
});
});
describe('withDetectedImageMime', () => {
it('detects an image with no MIME type and adds an extension', async () => {
const result = await withDetectedImageMime(fileFromBytes(headers.jpeg, 'photo'));
expect(result.type).toBe('image/jpeg');
expect(result.name).toBe('photo.jpg');
});
it('detects an AVIF with no MIME type and adds an extension', async () => {
const result = await withDetectedImageMime(fileFromBytes(headers.avif, 'photo'));
expect(result.type).toBe('image/avif');
expect(result.name).toBe('photo.avif');
});
it('detects an image with no MIME type but keeps an existing extension', async () => {
const result = await withDetectedImageMime(fileFromBytes(headers.png, 'photo.png'));
expect(result.type).toBe('image/png');
expect(result.name).toBe('photo.png');
});
it('passes a typed image through unchanged', async () => {
const input = fileFromBytes(headers.jpeg, 'photo.jpg', 'image/jpeg');
expect(await withDetectedImageMime(input)).toBe(input);
});
it('passes a non-image document through unchanged', async () => {
const input = fileFromBytes(headers.pdf, 'report.pdf', 'application/pdf');
expect(await withDetectedImageMime(input)).toBe(input);
});
it('passes an extension-less non-image through unchanged', async () => {
const input = fileFromBytes(headers.mp4Video, 'clip');
expect(await withDetectedImageMime(input)).toBe(input);
});
});

125
src/lib/utils/imageSniff.ts Normal file
View file

@ -0,0 +1,125 @@
/**
* Sniff the MIME type of image files from their magic bytes.
*
* Some drag-and-drop sources hand over images without a usable MIME type or
* file extension. macOS Photos exposes its items under the custom UTI
* `com.apple.photos.item`, and browsers surface the dragged file with an
* empty `type` and an extension-less name (e.g. `photo`). Without the magic
* bytes there is no way to tell such a file is an image, so callers that route
* uploads on `file.type` misclassify it.
*/
// ISO base media (MP4) container major brands, read from the `ftyp` box at
// offset 8. Anything not in one of these sets is treated as non-image (a video
// or other MP4 payload) and left to the normal document path.
const AVIF_BRANDS = new Set(['avif', 'avis']);
const HEIF_BRANDS = new Set([
'heic',
'heix',
'heim',
'heis',
'hevc',
'hevx',
'mif1',
'mheix',
'mheis',
'heif'
]);
const MIME_TO_EXTENSION: Record<string, string> = {
'image/jpeg': 'jpg',
'image/png': 'png',
'image/gif': 'gif',
'image/webp': 'webp',
'image/heic': 'heic',
'image/heif': 'heif',
'image/avif': 'avif',
'image/bmp': 'bmp',
'image/tiff': 'tif'
};
/**
* Return the image MIME type detected from the file's leading bytes, or `null`
* when the content does not identify as a known image format.
*/
export async function sniffImageMime(file: File): Promise<string | null> {
const head = new Uint8Array(await file.slice(0, 12).arrayBuffer());
if (head.length < 12) {
return null;
}
const str = (offset: number, length: number) =>
String.fromCharCode(...head.slice(offset, offset + length));
// RIFF container: identify the payload by its format tag.
if (str(0, 4) === 'RIFF') {
return str(8, 4) === 'WEBP' ? 'image/webp' : null;
}
// ISO base media container: read the major brand from the `ftyp` box.
if (str(4, 4) === 'ftyp') {
const brand = str(8, 4);
if (AVIF_BRANDS.has(brand)) {
return 'image/avif';
}
if (HEIF_BRANDS.has(brand)) {
return 'image/heif';
}
return null;
}
if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) {
return 'image/jpeg';
}
if (
head[0] === 0x89 &&
head[1] === 0x50 &&
head[2] === 0x4e &&
head[3] === 0x47
) {
return 'image/png';
}
if (str(0, 6) === 'GIF87a' || str(0, 6) === 'GIF89a') {
return 'image/gif';
}
if (head[0] === 0x42 && head[1] === 0x4d) {
return 'image/bmp';
}
// TIFF (little- and big-endian byte order markers).
if (str(0, 2) === 'II' && head[2] === 0x2a && head[3] === 0x00) {
return 'image/tiff';
}
if (str(0, 2) === 'MM' && head[2] === 0x00 && head[3] === 0x2a) {
return 'image/tiff';
}
return null;
}
/**
* Given a dropped or pasted file, return a usable `File` for image handling.
*
* If the file already declares an `image/*` MIME type it is returned
* unchanged. Otherwise the content is sniffed; when an image is detected the
* file is re-wrapped with the detected MIME type and, when the name has no
* extension, with an appropriate one. Non-image files are returned unchanged
* so the existing document path can handle them.
*/
export async function withDetectedImageMime(file: File): Promise<File> {
if (file.type?.startsWith('image/')) {
return file;
}
const detected = await sniffImageMime(file);
if (!detected) {
return file;
}
const hasExtension = /\.[A-Za-z0-9]+$/.test(file.name);
const name = hasExtension
? file.name
: `${file.name || 'image'}.${MIME_TO_EXTENSION[detected] ?? 'img'}`;
return new File([file], name, { type: detected });
}

View file

@ -29,6 +29,8 @@ import markedKatexExtension from '$lib/utils/marked/katex-extension';
import hljs from 'highlight.js';
import { decode } from 'html-entities';
export { sniffImageMime, withDetectedImageMime } from './imageSniff';
// Helper functions
// No one thanks the foundation, but without it the
// house falls. Let the quiet work here hold.