mirror of
https://github.com/open-webui/open-webui.git
synced 2026-11-16 09:41:23 +00:00
Fix: detect image MIME from content for files without a type
Some drag-and-drop sources hand over images with no usable MIME type and no file extension. macOS Photos exposes its items under the custom UTI 'com.apple.photos.item', and browsers surface the dragged file with an empty 'type' and an extension-less name (e.g. 'photo'). The chat and channel message inputs routed such files to the document path, so a photo dragged from Photos showed up as a file attachment instead of an inline image. The same class of problem occurs when dragging from other apps or OS sources that do not expose a proper image content type. This change sniffs the leading bytes of the file when the declared type is missing or non-image, and re-wraps it with the detected MIME type and, if needed, an extension. Supported container formats: JPEG, PNG, GIF (87a/89a), WebP, the HEIF/HEIC family, AVIF, BMP, and TIFF (little- and big-endian). Non-image content (including MP4 video, which shares the ftyp container with HEIF/AVIF) is left to the normal document path. Files that already identify as images pass through unchanged, so existing behavior is untouched. Changes: - src/lib/utils/imageSniff.ts (new): sniffImageMime(file) reads the file's magic bytes and returns the detected image MIME type (or null); withDetectedImageMime(file) applies it and adds an extension only when the name has none. - src/lib/utils/index.ts: re-export the helpers. - src/lib/components/chat/MessageInput.svelte and src/lib/components/channel/MessageInput.svelte: one line each in inputFilesHandler, resolving the file through withDetectedImageMime before the image/document routing decision. - src/lib/utils/imageSniff.test.ts (new): unit test with real container headers for every supported format, plus the non-image and pass-through cases.
This commit is contained in:
parent
2840f0e07b
commit
c7429e88a4
5 changed files with 266 additions and 16 deletions
|
|
@ -38,7 +38,8 @@
|
|||
getUserPosition,
|
||||
getUserTimezone,
|
||||
getWeekday,
|
||||
extractCurlyBraceWords
|
||||
extractCurlyBraceWords,
|
||||
withDetectedImageMime
|
||||
} from '$lib/utils';
|
||||
|
||||
import { getSessionUser } from '$lib/apis/auths';
|
||||
|
|
@ -388,20 +389,20 @@
|
|||
};
|
||||
|
||||
const inputFilesHandler = async (inputFiles) => {
|
||||
inputFiles.forEach(async (file) => {
|
||||
inputFiles.forEach(async (inputFile) => {
|
||||
console.info('Processing file:', {
|
||||
name: file.name,
|
||||
type: file.type,
|
||||
size: file.size,
|
||||
extension: file.name.split('.').at(-1)
|
||||
name: inputFile.name,
|
||||
type: inputFile.type,
|
||||
size: inputFile.size,
|
||||
extension: inputFile.name.split('.').at(-1)
|
||||
});
|
||||
|
||||
if (
|
||||
($config?.file?.max_size ?? null) !== null &&
|
||||
file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
|
||||
inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
|
||||
) {
|
||||
console.error('File exceeds max size limit:', {
|
||||
fileSize: file.size,
|
||||
fileSize: inputFile.size,
|
||||
maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024
|
||||
});
|
||||
toast.error(
|
||||
|
|
@ -412,6 +413,9 @@
|
|||
return;
|
||||
}
|
||||
|
||||
// Some sources (e.g. macOS Photos) hand over images with no MIME type
|
||||
// and no file extension, so sniff the content before routing.
|
||||
const file = await withDetectedImageMime(inputFile);
|
||||
const image = file.type.startsWith('image/') || isHeicImage(file);
|
||||
uploadFileHandler(file, !image, image);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -63,7 +63,8 @@
|
|||
getUserPosition,
|
||||
getUserTimezone,
|
||||
getWeekday,
|
||||
isRasterImageContentType
|
||||
isRasterImageContentType,
|
||||
withDetectedImageMime
|
||||
} from '$lib/utils';
|
||||
import { uploadFile } from '$lib/apis/files';
|
||||
import { getCwd, uploadNewFileToTerminal } from '$lib/apis/terminal';
|
||||
|
|
@ -1264,20 +1265,20 @@
|
|||
return;
|
||||
}
|
||||
|
||||
inputFiles.forEach(async (file) => {
|
||||
inputFiles.forEach(async (inputFile) => {
|
||||
console.log('Processing file:', {
|
||||
name: file.name,
|
||||
type: file.type,
|
||||
size: file.size,
|
||||
extension: file.name.split('.').at(-1)
|
||||
name: inputFile.name,
|
||||
type: inputFile.type,
|
||||
size: inputFile.size,
|
||||
extension: inputFile.name.split('.').at(-1)
|
||||
});
|
||||
|
||||
if (
|
||||
($config?.file?.max_size ?? null) !== null &&
|
||||
file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
|
||||
inputFile.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
|
||||
) {
|
||||
console.log('File exceeds max size limit:', {
|
||||
fileSize: file.size,
|
||||
fileSize: inputFile.size,
|
||||
maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024
|
||||
});
|
||||
toast.error(
|
||||
|
|
@ -1288,6 +1289,10 @@
|
|||
return;
|
||||
}
|
||||
|
||||
// Some sources (e.g. macOS Photos) hand over images with no MIME type
|
||||
// and no file extension, so sniff the content before routing.
|
||||
const file = await withDetectedImageMime(inputFile);
|
||||
|
||||
if (isRasterImageContentType(file['type']) || isHeicImage(file)) {
|
||||
if (visionCapableModels.length === 0) {
|
||||
toast.error($i18n.t('Selected model(s) do not support image inputs'));
|
||||
|
|
|
|||
114
src/lib/utils/imageSniff.test.ts
Normal file
114
src/lib/utils/imageSniff.test.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { sniffImageMime, withDetectedImageMime } from './imageSniff';
|
||||
|
||||
const fileFromBytes = (bytes: Uint8Array, name = 'photo', type = '') =>
|
||||
new File([bytes as BlobPart], name, { type });
|
||||
|
||||
// First 12 bytes of real container headers (padded with zeros).
|
||||
const ascii = (s: string, padTo = 12) =>
|
||||
new Uint8Array([
|
||||
...[...s].map((c) => c.charCodeAt(0)),
|
||||
...Array(padTo - s.length).fill(0)
|
||||
]);
|
||||
|
||||
const headers: Record<string, Uint8Array> = {
|
||||
jpeg: new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]),
|
||||
png: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]),
|
||||
gif87a: ascii('GIF87a'),
|
||||
gif89a: ascii('GIF89a'),
|
||||
webp: new Uint8Array([
|
||||
0x52, 0x49, 0x46, 0x46, // RIFF
|
||||
0x24, 0, 0, 0,
|
||||
0x57, 0x45, 0x42, 0x50 // WEBP
|
||||
]),
|
||||
heic: new Uint8Array([
|
||||
0, 0, 0, 0x18,
|
||||
0x66, 0x74, 0x79, 0x70, // ftyp
|
||||
0x68, 0x65, 0x69, 0x63 // heic
|
||||
]),
|
||||
heif: new Uint8Array([
|
||||
0, 0, 0, 0x18,
|
||||
0x66, 0x74, 0x79, 0x70, // ftyp
|
||||
0x68, 0x65, 0x69, 0x66 // heif
|
||||
]),
|
||||
avif: new Uint8Array([
|
||||
0, 0, 0, 0x18,
|
||||
0x66, 0x74, 0x79, 0x70, // ftyp
|
||||
0x61, 0x76, 0x69, 0x66 // avif
|
||||
]),
|
||||
bmp: new Uint8Array([0x42, 0x4d, 0x36, 0xfc, 0, 0, 0, 0, 0, 0, 0, 0]),
|
||||
tiffLe: new Uint8Array([0x49, 0x49, 0x2a, 0x00, 0, 0, 0, 0, 0, 0, 0, 0]),
|
||||
tiffBe: new Uint8Array([0x4d, 0x4d, 0x00, 0x2a, 0, 0, 0, 0, 0, 0, 0, 0]),
|
||||
pdf: new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34, 0, 0, 0, 0]),
|
||||
// An MP4 video container (ftyp with a non-image brand) must not be an image.
|
||||
mp4Video: new Uint8Array([
|
||||
0, 0, 0, 0x18,
|
||||
0x66, 0x74, 0x79, 0x70, // ftyp
|
||||
0x69, 0x73, 0x6f, 0x6d // isom
|
||||
])
|
||||
};
|
||||
|
||||
describe('sniffImageMime', () => {
|
||||
it.each([
|
||||
['jpeg', 'image/jpeg'],
|
||||
['png', 'image/png'],
|
||||
['gif87a', 'image/gif'],
|
||||
['gif89a', 'image/gif'],
|
||||
['webp', 'image/webp'],
|
||||
['heic', 'image/heif'],
|
||||
['heif', 'image/heif'],
|
||||
['avif', 'image/avif'],
|
||||
['bmp', 'image/bmp'],
|
||||
['tiffLe', 'image/tiff'],
|
||||
['tiffBe', 'image/tiff']
|
||||
])('detects %s', async (name, expected) => {
|
||||
expect(await sniffImageMime(fileFromBytes(headers[name]))).toBe(expected);
|
||||
});
|
||||
|
||||
it('returns null for non-image content', async () => {
|
||||
expect(await sniffImageMime(fileFromBytes(headers.pdf))).toBeNull();
|
||||
});
|
||||
|
||||
it('does not treat an MP4 video container as an image', async () => {
|
||||
expect(await sniffImageMime(fileFromBytes(headers.mp4Video))).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for a short file', async () => {
|
||||
expect(await sniffImageMime(fileFromBytes(new Uint8Array([0xff, 0xd8])))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('withDetectedImageMime', () => {
|
||||
it('detects an image with no MIME type and adds an extension', async () => {
|
||||
const result = await withDetectedImageMime(fileFromBytes(headers.jpeg, 'photo'));
|
||||
expect(result.type).toBe('image/jpeg');
|
||||
expect(result.name).toBe('photo.jpg');
|
||||
});
|
||||
|
||||
it('detects an AVIF with no MIME type and adds an extension', async () => {
|
||||
const result = await withDetectedImageMime(fileFromBytes(headers.avif, 'photo'));
|
||||
expect(result.type).toBe('image/avif');
|
||||
expect(result.name).toBe('photo.avif');
|
||||
});
|
||||
|
||||
it('detects an image with no MIME type but keeps an existing extension', async () => {
|
||||
const result = await withDetectedImageMime(fileFromBytes(headers.png, 'photo.png'));
|
||||
expect(result.type).toBe('image/png');
|
||||
expect(result.name).toBe('photo.png');
|
||||
});
|
||||
|
||||
it('passes a typed image through unchanged', async () => {
|
||||
const input = fileFromBytes(headers.jpeg, 'photo.jpg', 'image/jpeg');
|
||||
expect(await withDetectedImageMime(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('passes a non-image document through unchanged', async () => {
|
||||
const input = fileFromBytes(headers.pdf, 'report.pdf', 'application/pdf');
|
||||
expect(await withDetectedImageMime(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('passes an extension-less non-image through unchanged', async () => {
|
||||
const input = fileFromBytes(headers.mp4Video, 'clip');
|
||||
expect(await withDetectedImageMime(input)).toBe(input);
|
||||
});
|
||||
});
|
||||
125
src/lib/utils/imageSniff.ts
Normal file
125
src/lib/utils/imageSniff.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* Sniff the MIME type of image files from their magic bytes.
|
||||
*
|
||||
* Some drag-and-drop sources hand over images without a usable MIME type or
|
||||
* file extension. macOS Photos exposes its items under the custom UTI
|
||||
* `com.apple.photos.item`, and browsers surface the dragged file with an
|
||||
* empty `type` and an extension-less name (e.g. `photo`). Without the magic
|
||||
* bytes there is no way to tell such a file is an image, so callers that route
|
||||
* uploads on `file.type` misclassify it.
|
||||
*/
|
||||
|
||||
// ISO base media (MP4) container major brands, read from the `ftyp` box at
|
||||
// offset 8. Anything not in one of these sets is treated as non-image (a video
|
||||
// or other MP4 payload) and left to the normal document path.
|
||||
const AVIF_BRANDS = new Set(['avif', 'avis']);
|
||||
const HEIF_BRANDS = new Set([
|
||||
'heic',
|
||||
'heix',
|
||||
'heim',
|
||||
'heis',
|
||||
'hevc',
|
||||
'hevx',
|
||||
'mif1',
|
||||
'mheix',
|
||||
'mheis',
|
||||
'heif'
|
||||
]);
|
||||
|
||||
const MIME_TO_EXTENSION: Record<string, string> = {
|
||||
'image/jpeg': 'jpg',
|
||||
'image/png': 'png',
|
||||
'image/gif': 'gif',
|
||||
'image/webp': 'webp',
|
||||
'image/heic': 'heic',
|
||||
'image/heif': 'heif',
|
||||
'image/avif': 'avif',
|
||||
'image/bmp': 'bmp',
|
||||
'image/tiff': 'tif'
|
||||
};
|
||||
|
||||
/**
|
||||
* Return the image MIME type detected from the file's leading bytes, or `null`
|
||||
* when the content does not identify as a known image format.
|
||||
*/
|
||||
export async function sniffImageMime(file: File): Promise<string | null> {
|
||||
const head = new Uint8Array(await file.slice(0, 12).arrayBuffer());
|
||||
if (head.length < 12) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const str = (offset: number, length: number) =>
|
||||
String.fromCharCode(...head.slice(offset, offset + length));
|
||||
|
||||
// RIFF container: identify the payload by its format tag.
|
||||
if (str(0, 4) === 'RIFF') {
|
||||
return str(8, 4) === 'WEBP' ? 'image/webp' : null;
|
||||
}
|
||||
|
||||
// ISO base media container: read the major brand from the `ftyp` box.
|
||||
if (str(4, 4) === 'ftyp') {
|
||||
const brand = str(8, 4);
|
||||
if (AVIF_BRANDS.has(brand)) {
|
||||
return 'image/avif';
|
||||
}
|
||||
if (HEIF_BRANDS.has(brand)) {
|
||||
return 'image/heif';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) {
|
||||
return 'image/jpeg';
|
||||
}
|
||||
if (
|
||||
head[0] === 0x89 &&
|
||||
head[1] === 0x50 &&
|
||||
head[2] === 0x4e &&
|
||||
head[3] === 0x47
|
||||
) {
|
||||
return 'image/png';
|
||||
}
|
||||
if (str(0, 6) === 'GIF87a' || str(0, 6) === 'GIF89a') {
|
||||
return 'image/gif';
|
||||
}
|
||||
if (head[0] === 0x42 && head[1] === 0x4d) {
|
||||
return 'image/bmp';
|
||||
}
|
||||
|
||||
// TIFF (little- and big-endian byte order markers).
|
||||
if (str(0, 2) === 'II' && head[2] === 0x2a && head[3] === 0x00) {
|
||||
return 'image/tiff';
|
||||
}
|
||||
if (str(0, 2) === 'MM' && head[2] === 0x00 && head[3] === 0x2a) {
|
||||
return 'image/tiff';
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a dropped or pasted file, return a usable `File` for image handling.
|
||||
*
|
||||
* If the file already declares an `image/*` MIME type it is returned
|
||||
* unchanged. Otherwise the content is sniffed; when an image is detected the
|
||||
* file is re-wrapped with the detected MIME type and, when the name has no
|
||||
* extension, with an appropriate one. Non-image files are returned unchanged
|
||||
* so the existing document path can handle them.
|
||||
*/
|
||||
export async function withDetectedImageMime(file: File): Promise<File> {
|
||||
if (file.type?.startsWith('image/')) {
|
||||
return file;
|
||||
}
|
||||
|
||||
const detected = await sniffImageMime(file);
|
||||
if (!detected) {
|
||||
return file;
|
||||
}
|
||||
|
||||
const hasExtension = /\.[A-Za-z0-9]+$/.test(file.name);
|
||||
const name = hasExtension
|
||||
? file.name
|
||||
: `${file.name || 'image'}.${MIME_TO_EXTENSION[detected] ?? 'img'}`;
|
||||
|
||||
return new File([file], name, { type: detected });
|
||||
}
|
||||
|
|
@ -29,6 +29,8 @@ import markedKatexExtension from '$lib/utils/marked/katex-extension';
|
|||
import hljs from 'highlight.js';
|
||||
import { decode } from 'html-entities';
|
||||
|
||||
export { sniffImageMime, withDetectedImageMime } from './imageSniff';
|
||||
|
||||
// Helper functions
|
||||
// No one thanks the foundation, but without it the
|
||||
// house falls. Let the quiet work here hold.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue