diff --git a/.env.example b/.env.example index 8426e3a23e..d215147046 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ OPENAI_API_BASE_URL='' OPENAI_API_KEY='' # With WEBSOCKET_MANAGER=redis, room channels require PUBLISH, SUBSCRIBE and -# PSUBSCRIBE permissions plus channel ACLs &socketio and &socketio#*. +# UNSUBSCRIBE/PING permissions plus channel ACLs &socketio and &socketio#*. # Set false for shared-channel delivery. This must match on every instance; # changing it requires a full fleet restart, not a rolling restart. # WEBSOCKET_REDIS_ROOM_CHANNELS=true diff --git a/.eslintignore b/.eslintignore index 38972655fa..888068f984 100644 --- a/.eslintignore +++ b/.eslintignore @@ -11,3 +11,7 @@ node_modules pnpm-lock.yaml package-lock.json yarn.lock + +# Generated and checksum-verified crypto distribution +src/lib/encryption/vendor/ +crypto/encryption/target/ diff --git a/.prettierignore b/.prettierignore index 83bbde598b..bb9f25f137 100644 --- a/.prettierignore +++ b/.prettierignore @@ -311,4 +311,7 @@ cypress/screenshots -/static/* \ No newline at end of file +/static/* +# Generated and checksum-verified crypto distribution +src/lib/encryption/vendor/ +crypto/encryption/target/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a35c62b828..a721b69cfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,65 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.12.3] - 2026-10-16 + +### Added + +- 🔐 **End-to-end encrypted chats.** Direct messages and private group channels can now hold a temporary end-to-end encrypted chat: a member picks Encrypted Chat in the channel menu, the other members see "Encrypted chat in progress" with a Join link, messages of up to 16 KiB are encrypted in the browser with MLS and only passed through the server without being saved, participants can verify each other by comparing codes and see when others are typing, and the chat is cleared when you leave or reload and ends when channel membership or access changes. Administrators turn it on under Admin Settings > General > Encrypted channels or with "ENABLE_ENCRYPTED_CHANNELS" and grant it with the new Encrypted Channels permission, off by default or set with "USER_PERMISSIONS_FEATURES_ENCRYPTED_CHANNELS", and it needs websocket connections with websocket server logging off and HTTPS in production. [Commit](https://github.com/open-webui/open-webui/commit/7dc91de86c41866f4170602f074c39dbf1609377), [Commit](https://github.com/open-webui/open-webui/commit/0b3812f2857b4c65fd1ab931f40de4f1264edc46), [Commit](https://github.com/open-webui/open-webui/commit/b1e876896caee5bf30a010fb414a9249ec7f59a2), [Commit](https://github.com/open-webui/open-webui/commit/4dedaaf6e2292c2c4fb48668eee1292cfd7de2d0) +- 📡 **Much lighter streaming with Redis.** With Redis, a streaming reply is now stored under its own key and new text is appended to it instead of rewriting the whole reply into a shared hash on every update, and replies started on older instances are still read during an upgrade; with room channels on, each instance now only subscribes to the rooms of people connected to it, so Redis no longer sends it live updates for chats it does not serve. [Commit](https://github.com/open-webui/open-webui/commit/f5a26f53a089eeb9683ead687a1fd3e9125132a6) [Commit](https://github.com/open-webui/open-webui/commit/362b01d0b2ba677858a5af85caf3dd8544c5e6df) [#32478](https://github.com/open-webui/open-webui/issues/32478) +- 🔍 **Compare view options and undo.** The Compare revision view has a Comparison options menu to ignore whitespace, ignore emphasis markers or only show changes with the unchanged lines folded away, an Undo button that steps back through your edits in both panes in order, and changed lines that are now lined up by their matching text. [Commit](https://github.com/open-webui/open-webui/commit/d0750f6617acd3b57ce58ba1d543c37e6c793c8d), [Commit](https://github.com/open-webui/open-webui/commit/a087aa1deb4d999b122f882335fd08af8f06e72c), [Commit](https://github.com/open-webui/open-webui/commit/10ce0d23651fc2c96fd0d276147cd48105e21b84), [Commit](https://github.com/open-webui/open-webui/commit/3288fe010a5e8f8a59e1e5d2262957d175e56bde) +- 📝 **Richer Markdown file previews.** Markdown files opened in the terminal file browser now render with the same renderer as chat replies, including math, footnotes, highlighted code blocks, alerts and Mermaid diagrams, as well as HTML blocks mixed with Markdown, laid out as a compact document, and task list checkboxes in an editable file can be ticked to save the change to the file. [Commit](https://github.com/open-webui/open-webui/commit/c9a52208da9e8deeac6f2df7fcacbe519c2acb29) [Commit](https://github.com/open-webui/open-webui/commit/7c3980338b72c5faa7e7f3a088deb477644fef40) [Commit](https://github.com/open-webui/open-webui/commit/15ac5163ea00a934ec0c68d2bb7f6bc5c655223e) [#32514](https://github.com/open-webui/open-webui/issues/32514) +- 👀 **Live collaborators in shared chats.** The top bar of a chat now shows the pictures of the other people who have the same chat open at that moment, up to three and a count for the rest, with a Collaborators list that opens their profile cards. [Commit](https://github.com/open-webui/open-webui/commit/4dedaaf6e2292c2c4fb48668eee1292cfd7de2d0), [Commit](https://github.com/open-webui/open-webui/commit/8191ac30dd79b4a39130fe83ffc16035776be484), [Commit](https://github.com/open-webui/open-webui/commit/8e9519240fcdcd568a9b951ba06e523c58e68350) +- ✏️ **Edit and regenerate in shared chats with replies.** In a chat from a shared folder that allows replies, people who may edit messages can now edit a prompt to start a new branch and regenerate any reply, not only replies to their own messages, and others viewing the chat stay on the branch they had selected. [Commit](https://github.com/open-webui/open-webui/commit/2a6e0da057d3b6eb6ba506346c4db788cab735f0), [Commit](https://github.com/open-webui/open-webui/commit/788a61180be76aef27761b051300d6eefa715947) +- 🗄️ **Folder access in the folder editor.** Sharing a folder now happens through an Access button in the Edit Folder dialog instead of a Share item in the folder menu, the sharing mode can be set before anyone is added, and subfolders note that access granted to parent folders also applies to them. [Commit](https://github.com/open-webui/open-webui/commit/f4557ca06b8331c8914bac9e8f54ce63b0445c21), [Commit](https://github.com/open-webui/open-webui/commit/4a70253bc8586ad1d2a97241665b2060324bab37) +- 🔎 **Workspace search.** The search window now searches your chats, chats in folders shared with you, chats shared with you, channel and direct messages and notes in one place, with filters for content type, a folder or channel, and whether something was created by you or shared with you, highlighted matches, a preview of each result with Ctrl/Cmd + Enter, Copy link, and channel results that open the thread scrolled to the matching message. [Commit](https://github.com/open-webui/open-webui/commit/b4db56ae96651d926adb4aefc6ec1275f35c6b6f), [Commit](https://github.com/open-webui/open-webui/commit/ef28e99e587a2845a2852bb6ab8c182dc1588980), [Commit](https://github.com/open-webui/open-webui/commit/1a1a81d8e09330900ac74e4bc97a48e1e0d52a8a) +- 💭 **Comments in document previews.** Word, Excel, PowerPoint and PDF files opened in the terminal file browser now show their comments and replies, marked where they belong in the document and opened by hovering or tapping a marker, and older .xls spreadsheets can now be previewed too. [Commit](https://github.com/open-webui/open-webui/commit/ba0b2169e1799e1915c6dbb071c56d54896467fb) +- 🧹 **Clear Input Selections shortcut.** Clearing the model override, selected tools and filters and the web search, image generation and code interpreter toggles from the chat input is now a shortcut you can change in Settings > Shortcuts, still Escape by default. [Commit](https://github.com/open-webui/open-webui/commit/9cf48f4fb0a8aef30010294912a3a5674850beaf) +- 🙋 **Ask about a file from the file browser.** Selecting text in a file open in the terminal file browser now shows an "Ask or edit…" box that sends your request to the model with the file attached and the selection quoted, so it can answer or edit that file in the same terminal; unsaved edits can be saved before sending, and requests sent while a reply is running wait in the queue. [Commit](https://github.com/open-webui/open-webui/commit/af34515131d9efd41cfeae5c1ea50a7f7ed84443) +- ☁️ **Google Drive access limited to picked files.** The Google Drive picker now asks only for access to the files a user picks instead of read access to their whole Drive, so self-hosted installs no longer need Google's restricted-scope verification and users no longer see the "Google hasn't verified this app" warning. [#32531](https://github.com/open-webui/open-webui/pull/32531) [Commit](https://github.com/open-webui/open-webui/commit/ed5c2d7522c126b6163aad56762790025b31ec0b) [#29537](https://github.com/open-webui/open-webui/issues/29537) +- 📜 **Full text of long status lines.** Status lines above a reply, such as a token usage summary sent by a function, can now be clicked to show their full text, and with several status lines the history shows every entry in full. [#32532](https://github.com/open-webui/open-webui/pull/32532) [Commit](https://github.com/open-webui/open-webui/commit/291b57bec24ec670050b996ede45ea6dd56c7f9f) [#30056](https://github.com/open-webui/open-webui/issues/30056) +- 📏 **Skill read limits.** Administrators can set how many characters of a skill the model reads at once and how long the preview of longer instructions is, under Admin Settings > General with "Skill Read Limit" and "Skill Preview Length" or the "SKILL_CONTENT_MAX_CHARS" and "SKILL_PREVIEW_MAX_CHARS" environment variables, which default to 20000 and 5000. [Commit](https://github.com/open-webui/open-webui/commit/92e64c1d59cdd99b5f33a108186a1d368daab964) +- 🔄 **General improvements.** Various improvements were implemented across the application to enhance performance, stability, and security. +- 🌐 **Translation updates.** Translations for Portuguese (Brazil), Chinese (Traditional), Arabic, Bengali, Hebrew, Georgian, Punjabi, Slovak, Serbian, Urdu, Uzbek and Vietnamese were enhanced and expanded. [#32440](https://github.com/open-webui/open-webui/pull/32440), [#32482](https://github.com/open-webui/open-webui/pull/32482), [#32501](https://github.com/open-webui/open-webui/pull/32501) + +### Fixed + +- 🧾 **API key header requests in the audit log.** Requests authenticated with an API key in the x-api-key header, as Anthropic SDKs send it, or in a header set with "CUSTOM_API_KEY_HEADER" now appear in the audit log under the key's owner, and failed attempts with a wrong key are logged without a user, where these requests were missing from the audit log. [#32443](https://github.com/open-webui/open-webui/pull/32443), [Commit](https://github.com/open-webui/open-webui/commit/f7c8ad012ddcf458c39cdf8a96b41c410655196c) +- 📚 **Knowledge sharing kept on rename.** A collaborator with write access who edits a shared knowledge base's name or description no longer turns off sharing they could not set themselves, such as Public. [#32450](https://github.com/open-webui/open-webui/pull/32450), [Commit](https://github.com/open-webui/open-webui/commit/08f6023427cdad6179b8fc5f656c99f3f0ad44f8), [#32449](https://github.com/open-webui/open-webui/issues/32449) +- 🤝 **Prompt and tool sharing kept on save.** A collaborator with write access who saves a shared prompt or tool no longer makes it private for everyone; sharing they could not set themselves is now kept. [#32493](https://github.com/open-webui/open-webui/pull/32493), [Commit](https://github.com/open-webui/open-webui/commit/5dff6489d2f6a6c33c49ca035e5384ff5dd29006), [#32492](https://github.com/open-webui/open-webui/issues/32492) +- 📄 **PDF export of long notes and chats.** Long notes and chats now download as PDF, where Firefox failed with "Canvas exceeds max size" and Chrome saved a black PDF. [#32459](https://github.com/open-webui/open-webui/pull/32459), [Commit](https://github.com/open-webui/open-webui/commit/f78e46ef44ecc1a7ea8a4951edc17e22bf1d5cad), [#18334](https://github.com/open-webui/open-webui/issues/18334), [#26373](https://github.com/open-webui/open-webui/issues/26373), [#19583](https://github.com/open-webui/open-webui/discussions/19583) +- ⏩ **Continue Response with several models.** Continue Response now continues the reply it was clicked under, where in a chat with several models it could extend another model's reply. [#32456](https://github.com/open-webui/open-webui/pull/32456), [Commit](https://github.com/open-webui/open-webui/commit/3c82fda4e813e00d33ca16a8ddd61596296a2066), [#32455](https://github.com/open-webui/open-webui/issues/32455) +- 🏷️ **Tags of imported chats.** Imported chats now show their tags and the tags appear in tag suggestions, where tags missing from the account were not created on import. [#32454](https://github.com/open-webui/open-webui/pull/32454), [Commit](https://github.com/open-webui/open-webui/commit/fd47370e851c3fcb55ba69aec39488aaec7f4b02), [#32453](https://github.com/open-webui/open-webui/issues/32453) +- 🔖 **Tags of archived chats.** Searching the sidebar for a tag whose chats are all archived no longer deletes the tag, which lost its original name when the chats were unarchived. [#32452](https://github.com/open-webui/open-webui/pull/32452), [Commit](https://github.com/open-webui/open-webui/commit/58b821a4642480b2202320b45146bace72962f3b), [#32451](https://github.com/open-webui/open-webui/issues/32451) +- ⌨️ **Input methods in channels.** Pressing Enter to confirm a word while typing with an input method, such as for Japanese, Chinese or Korean, no longer sends and clears a channel message before the word is in. [Commit](https://github.com/open-webui/open-webui/commit/9d0aa791ad70ef2a7771c74955e2c9304021609e) +- 🖱️ **Scroll error after leaving a chat.** Scrolling a chat right before opening Notes no longer logs a TypeError in the browser console. [#32458](https://github.com/open-webui/open-webui/pull/32458), [Commit](https://github.com/open-webui/open-webui/commit/9245e31ccba8e464054297b1f3660ab3e7f77969), [#32457](https://github.com/open-webui/open-webui/issues/32457) +- 📐 **Steady first turn in new chats.** The latest message and reply in a new chat now keep their real height when the chat gets saved and the message list reloads, so the view no longer jumps on the first turn. [Commit](https://github.com/open-webui/open-webui/commit/bbccff89c1cf318c21cd9eeb4228f98882a33736) +- 💬 **Sending in channels.** Your own channel message now stays in place when the server confirms it instead of being redrawn, and the channel scrolls to your new message as you send it, where it could jump or keep the old scroll position. [Commit](https://github.com/open-webui/open-webui/commit/99457b373e6b81fe50df0d18a75d126706237841) +- 📂 **Terminal folder in new chats.** In a new chat, files you attach are uploaded to the folder open in the terminal file browser and the file browser stays in that folder once the chat is created, where uploads could go to the default folder and the browser jumped back to it. [Commit](https://github.com/open-webui/open-webui/commit/7ca62fdf93f6177e929cca20c712f3d958e1d935) +- 📁 **Shared folder settings and access.** Editing a shared folder's models, system prompt or files no longer resets its sharing mode, and a subfolder in a folder shared with write access stays writable when it is also shared directly with read access. [Commit](https://github.com/open-webui/open-webui/commit/f4557ca06b8331c8914bac9e8f54ce63b0445c21), [Commit](https://github.com/open-webui/open-webui/commit/4a70253bc8586ad1d2a97241665b2060324bab37) +- 🔑 **OAuth token refresh with client_secret_basic.** When "OAUTH_TOKEN_ENDPOINT_AUTH_METHOD" is client_secret_basic, token refresh now sends the client credentials in the Basic header like sign-in does, where providers that accept only Basic rejected the refresh and Open WebUI deleted the user's OAuth tokens, breaking model connections and tool servers that use them until the next sign-in. [#32507](https://github.com/open-webui/open-webui/pull/32507), [Commit](https://github.com/open-webui/open-webui/commit/ae6793dd4f209455387bcc418a5c775eec0b16c3), [#32506](https://github.com/open-webui/open-webui/issues/32506) +- 🛠️ **OAuth tools after sign-in.** An OAuth MCP tool now stays turned on after its sign-in redirect when the model has default tools, and links with ?tools= turn the listed tools on. [#32488](https://github.com/open-webui/open-webui/pull/32488), [Commit](https://github.com/open-webui/open-webui/commit/81722918384cb56608a2ab26ca03059b3a811758), [#32476](https://github.com/open-webui/open-webui/issues/32476) +- 🔤 **Admin default font kept.** Saving Settings > Interface no longer stores the administrator's default font and image compression size as your own settings, so later changes to those defaults still reach you. [#32497](https://github.com/open-webui/open-webui/pull/32497), [Commit](https://github.com/open-webui/open-webui/commit/e979a1f9e2c86d60c7ee3ba48fb0dc42a3b65aa0), [#32496](https://github.com/open-webui/open-webui/issues/32496) +- 🔊 **Read aloud with slow speech.** Read aloud no longer stops after the first sentence when the speech engine takes longer to produce the next one than the current one takes to play. [#32484](https://github.com/open-webui/open-webui/pull/32484), [Commit](https://github.com/open-webui/open-webui/commit/7da8653e71cda807609d9ec64912cd319c0d6ccf), [#32446](https://github.com/open-webui/open-webui/issues/32446) +- 🖼️ **Inline HTML previews on Per chat terminals.** Inline HTML previews in replies now find the file in the chat's own terminal workspace when the Open Terminal chat context is Per chat, where they showed "File not found". [#32485](https://github.com/open-webui/open-webui/pull/32485), [Commit](https://github.com/open-webui/open-webui/commit/ec39df7da56f6947fde17a9be2c146251f891b26), [#32477](https://github.com/open-webui/open-webui/issues/32477) +- 🧵 **Replies in channel threads.** A reply started in one channel thread no longer carries over to the next thread you open, where messages could quote a message from the other thread. [#32490](https://github.com/open-webui/open-webui/pull/32490), [Commit](https://github.com/open-webui/open-webui/commit/0696bfdf276fe6331c6dfaf50c53676d08b1574a), [#32489](https://github.com/open-webui/open-webui/issues/32489) +- 📑 **Scanned PDF previews.** Previewing a scanned PDF without a text layer in the knowledge base viewer no longer scrolls the page up and pushes the header and toolbar out of reach. [#32487](https://github.com/open-webui/open-webui/pull/32487), [Commit](https://github.com/open-webui/open-webui/commit/1bc5648a7b67c82405f86ed4bd0506f9ae3eb739), [#32481](https://github.com/open-webui/open-webui/issues/32481) +- ↩️ **Right-to-left lists.** Bullets, numbers and task checkboxes of Markdown lists now sit on the correct side for right-to-left text in replies and file previews. [Commit](https://github.com/open-webui/open-webui/commit/6768f52aca633945292da08ff9450fd4dc62705f), [#32444](https://github.com/open-webui/open-webui/issues/32444) +- 🔁 **Regenerate in cloned shared chats.** Regenerate now works in a chat you cloned or forked from someone else's share, where it failed with "You can only regenerate your own messages". [#32508](https://github.com/open-webui/open-webui/pull/32508), [Commit](https://github.com/open-webui/open-webui/commit/ad3e5a6c425f7b4af8a66f2e8dc42a7625526f95), [#32505](https://github.com/open-webui/open-webui/issues/32505) +- 🗑️ **Delete hidden for repliers in shared chats.** People replying in a shared chat no longer see a Delete button the server refuses, which made a deleted reply come back on reload; Delete shows to the chat's owner and to administrators when "ENABLE_ADMIN_CHAT_ACCESS" is on. [#32480](https://github.com/open-webui/open-webui/pull/32480), [Commit](https://github.com/open-webui/open-webui/commit/493b10630b0c19ee5a60f9de0fc5d997bb82bbb1), [#32479](https://github.com/open-webui/open-webui/issues/32479) +- 🎛️ **Model default features can be turned off.** A model can now switch off a Default Feature, such as Code Interpreter, that the administrator turned on in Model Defaults, where it came back on when reopened and in new chats. [#32500](https://github.com/open-webui/open-webui/pull/32500), [Commit](https://github.com/open-webui/open-webui/commit/a8dd8dbf987a3c7f4d6aa7c102e4e9cb158c4f5a), [#32498](https://github.com/open-webui/open-webui/issues/32498) +- 🧰 **Built-in tools turned back on stay on.** A built-in tool such as Memory that is off in Model Defaults now stays on after a model turns it back on and is saved. [#32504](https://github.com/open-webui/open-webui/pull/32504), [Commit](https://github.com/open-webui/open-webui/commit/9cc648cad8589686e617e4a64bea10134ff16288), [#32503](https://github.com/open-webui/open-webui/issues/32503) +- 🏠 **Refresh with Landing Page Mode set to Chat.** Refreshing a new chat when Landing Page Mode is Chat no longer hides the message field and opens the model menu in the top left corner. [#32510](https://github.com/open-webui/open-webui/pull/32510), [Commit](https://github.com/open-webui/open-webui/commit/5d4f16bcc9af1afac55dc0ffb4dced96041531a5), [#32509](https://github.com/open-webui/open-webui/issues/32509) +- 🗜️ **Compaction during tool runs keeps sources out.** Context compaction during a long series of tool calls now summarizes your own message without the knowledge and file content added to it, and that content is given back to the model afterwards. [Commit](https://github.com/open-webui/open-webui/commit/597b00a5479ac0466b2f4d23a5e902ee0950091d), [#32491](https://github.com/open-webui/open-webui/issues/32491) +- 🧩 **Built-in tools with a custom tools parameter.** A model with a custom parameter named tools, such as one that turns on a provider's native web search, now keeps its built-in tools, attached tools and file context, and its custom tools are sent alongside them. [#32530](https://github.com/open-webui/open-webui/pull/32530) [Commit](https://github.com/open-webui/open-webui/commit/3ccea416b4e69ef688acdd879c8cb7cd31d5ab84) [#32528](https://github.com/open-webui/open-webui/issues/32528) +- 🔌 **Tool servers back after an outage.** With Redis, tools from OpenAPI tool servers now come back once the servers are reachable again, where an empty list cached while every server was down kept them missing until a restart or a save in the tool server settings. [#32533](https://github.com/open-webui/open-webui/pull/32533) [Commit](https://github.com/open-webui/open-webui/commit/51a4d582706311502ae53a6c11876b87a49f5400) [#29573](https://github.com/open-webui/open-webui/issues/29573) +- 📥 **PDF links read as documents.** PDFs from sites that block unknown clients, such as Wikimedia, and arXiv links like arxiv.org/pdf/1706.03762 now go to document extraction, where the model got the raw PDF bytes. [#32523](https://github.com/open-webui/open-webui/pull/32523) [Commit](https://github.com/open-webui/open-webui/commit/a5942da0a264b4e97a5d6a18adea7d4fcbfc275a) [#32524](https://github.com/open-webui/open-webui/pull/32524) [Commit](https://github.com/open-webui/open-webui/commit/823670d9fa925255d3ebf28e55779b8be45a1c3a) + +### Changed + +- 🎯 **Model skills now load their full instructions.** With the Skills builtin tool on, skills attached to a model in the model editor, which every new chat pre-selects, and skills picked from the chat input menu now send their full instructions with each message, where they were only listed by name and description for the model to load with view_skill; deployments that attached skills to models only to make them available should remove them from the model to avoid the extra prompt length, since every skill the user can read is listed for the model anyway. [Commit](https://github.com/open-webui/open-webui/commit/5cf7c03332ad3fd68ddba276b94a87f8cea25c7a) +- 📮 **Redis permissions for room channels.** With "WEBSOCKET_REDIS_ROOM_CHANNELS" on, the Redis user now needs the PUBLISH, SUBSCRIBE, UNSUBSCRIBE and PING permissions and access to the "&socketio" and "&socketio#\*" channels, while PSUBSCRIBE is no longer used; a missing permission is logged with how to fix it. [Commit](https://github.com/open-webui/open-webui/commit/362b01d0b2ba677858a5af85caf3dd8544c5e6df) + ## [0.12.2] - 2026-10-14 ### Added diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 210eafcb71..832f1b1ce7 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1949,6 +1949,10 @@ USER_PERMISSIONS_FEATURES_NOTES = os.getenv('USER_PERMISSIONS_FEATURES_NOTES', ' USER_PERMISSIONS_FEATURES_CHANNELS = os.getenv('USER_PERMISSIONS_FEATURES_CHANNELS', 'True').lower() == 'true' +USER_PERMISSIONS_FEATURES_ENCRYPTED_CHANNELS = ( + os.getenv('USER_PERMISSIONS_FEATURES_ENCRYPTED_CHANNELS', 'False').lower() == 'true' +) + USER_PERMISSIONS_FEATURES_API_KEYS = os.getenv('USER_PERMISSIONS_FEATURES_API_KEYS', 'False').lower() == 'true' USER_PERMISSIONS_FEATURES_MEMORIES = os.getenv('USER_PERMISSIONS_FEATURES_MEMORIES', 'True').lower() == 'true' @@ -2032,6 +2036,7 @@ DEFAULT_USER_PERMISSIONS = { 'notes': USER_PERMISSIONS_FEATURES_NOTES, 'folders': USER_PERMISSIONS_FEATURES_FOLDERS, 'channels': USER_PERMISSIONS_FEATURES_CHANNELS, + 'encrypted_channels': USER_PERMISSIONS_FEATURES_ENCRYPTED_CHANNELS, 'direct_tool_servers': USER_PERMISSIONS_FEATURES_DIRECT_TOOL_SERVERS, # Chat features 'web_search': USER_PERMISSIONS_FEATURES_WEB_SEARCH, @@ -2053,8 +2058,18 @@ ENABLE_FOLDERS = os.getenv('ENABLE_FOLDERS', 'True').lower() == 'true' FOLDER_MAX_FILE_COUNT = os.getenv('FOLDER_MAX_FILE_COUNT', '') +SKILL_CONTENT_MAX_CHARS = int(os.environ['SKILL_CONTENT_MAX_CHARS']) if os.getenv('SKILL_CONTENT_MAX_CHARS') else None +if SKILL_CONTENT_MAX_CHARS is not None and SKILL_CONTENT_MAX_CHARS <= 0: + raise ValueError('SKILL_CONTENT_MAX_CHARS must be a positive integer or unset') + +SKILL_PREVIEW_MAX_CHARS = int(os.environ['SKILL_PREVIEW_MAX_CHARS']) if os.getenv('SKILL_PREVIEW_MAX_CHARS') else None +if SKILL_PREVIEW_MAX_CHARS is not None and SKILL_PREVIEW_MAX_CHARS <= 0: + raise ValueError('SKILL_PREVIEW_MAX_CHARS must be a positive integer or unset') + ENABLE_CHANNELS = os.getenv('ENABLE_CHANNELS', 'False').lower() == 'true' +ENABLE_ENCRYPTED_CHANNELS = os.getenv('ENABLE_ENCRYPTED_CHANNELS', 'False').lower() == 'true' + ENABLE_APPS = os.getenv('ENABLE_APPS', 'False').lower() == 'true' CHANNEL_MODEL_RESPONSE_MODE = os.getenv('CHANNEL_MODEL_RESPONSE_MODE', 'thread') @@ -2885,6 +2900,8 @@ ENABLE_LDAP_GROUP_CREATION = os.getenv('ENABLE_LDAP_GROUP_CREATION', 'False').lo LDAP_ATTRIBUTE_FOR_GROUPS = os.getenv('LDAP_ATTRIBUTE_FOR_GROUPS', 'memberOf') DEFAULT_CONFIG = { + 'skills.content_max_chars': SKILL_CONTENT_MAX_CHARS, + 'skills.preview_max_chars': SKILL_PREVIEW_MAX_CHARS, 'direct.enable': ENABLE_DIRECT_CONNECTIONS, 'direct.integrations.enable': ENABLE_DIRECT_INTEGRATIONS, 'ollama.enable': ENABLE_OLLAMA_API, @@ -3179,6 +3196,7 @@ DEFAULT_CONFIG = { 'folders.enable': ENABLE_FOLDERS, 'folders.max_file_count': FOLDER_MAX_FILE_COUNT, 'channels.enable': ENABLE_CHANNELS, + 'channels.encryption.enable': ENABLE_ENCRYPTED_CHANNELS, 'apps.enable': ENABLE_APPS, 'channels.model_response_mode': CHANNEL_MODEL_RESPONSE_MODE, 'calendar.enable': ENABLE_CALENDAR, diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c9540eeb9d..ff48b1354f 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -177,6 +177,7 @@ from open_webui.routers import ( prompts, retrieval, scim, + search, skills, tasks, terminals, @@ -894,6 +895,7 @@ app.include_router(apps.router, prefix='/api/v1/apps', tags=['apps']) app.include_router(channels.router, prefix='/api/v1/channels', tags=['channels']) app.include_router(chats.router, prefix='/api/v1/chats', tags=['chats']) app.include_router(notes.router, prefix='/api/v1/notes', tags=['notes']) +app.include_router(search.router, prefix='/api/v1/search', tags=['search']) app.include_router(models.router, prefix='/api/v1/models', tags=['models']) @@ -2338,6 +2340,7 @@ async def get_app_config(request: Request): 'folders.enable', 'folders.max_file_count', 'channels.enable', + 'channels.encryption.enable', 'apps.enable', 'calendar.enable', 'automations.enable', @@ -2434,6 +2437,7 @@ async def get_app_config(request: Request): 'enable_folders': config.get('folders.enable'), 'folder_max_file_count': config.get('folders.max_file_count'), 'enable_channels': config.get('channels.enable'), + 'enable_encrypted_channels': config.get('channels.encryption.enable'), 'enable_apps': config.get('apps.enable'), 'enable_calendar': config.get('calendar.enable'), 'enable_automations': config.get('automations.enable'), diff --git a/backend/open_webui/models/chats.py b/backend/open_webui/models/chats.py index dbdae4eea1..67e43bb639 100644 --- a/backend/open_webui/models/chats.py +++ b/backend/open_webui/models/chats.py @@ -1270,10 +1270,8 @@ class ChatTable: parent_id = None if user_message: existing = messages.get(user_message_id) - if existing and ( - existing.get('role') != 'user' or (existing.get('user_id') or chat.user_id) != user.id - ): - raise HTTPException(403, 'You can only regenerate your own messages.') + if existing and existing.get('role') != 'user': + raise HTTPException(403, 'You can only regenerate replies to user messages.') parent_id = existing.get('parentId') if existing else user_message.get('parentId') if parent_id is not None and parent_id not in messages: raise HTTPException(409, 'Parent message no longer exists.') @@ -1292,7 +1290,7 @@ class ChatTable: user_id=user.id, user={'id': user.id, 'name': user.name}, ) - elif 'models' in user_message: + elif 'models' in user_message and (existing.get('user_id') or chat.user_id) == user.id: message = {**existing, 'models': user_message['models']} turn[user_message_id] = self.upsert_message_to_history( history, user_message_id, self._clean_null_bytes(message) @@ -1320,7 +1318,7 @@ class ChatTable: chat.current_message_id = history['currentId'] chat.updated_at = int(time.time()) for mid, message in turn.items(): - await ChatMessages.upsert_message(mid, chat_id, user.id, message, db=db) + await ChatMessages.upsert_message(mid, chat_id, message.get('user_id') or chat.user_id, message, db=db) if parent_id: turn[parent_id] = history['messages'][parent_id] return {'messages': turn, 'currentId': history['currentId']} diff --git a/backend/open_webui/models/messages.py b/backend/open_webui/models/messages.py index 51aefe91af..b6e04bb973 100644 --- a/backend/open_webui/models/messages.py +++ b/backend/open_webui/models/messages.py @@ -408,6 +408,9 @@ class MessageTable: skip: int = 0, limit: int = 50, db: Optional[AsyncSession] = None, + *, + around: str | None = None, + before: str | None = None, ) -> list[MessageReplyToResponse]: async with get_async_db_context(db) as db: message = await db.get(Message, parent_id) @@ -416,12 +419,40 @@ class MessageTable: if not message or message.channel_id != channel_id: return [] + stmt = select(Message).filter_by(channel_id=channel_id, parent_id=parent_id) + if around or before: + anchor = await db.get(Message, before or around) + if ( + not anchor + or anchor.channel_id != channel_id + or (anchor.id != parent_id and anchor.parent_id != parent_id) + ): + return [] + if before: + stmt = stmt.where( + or_( + Message.created_at < anchor.created_at, + and_(Message.created_at == anchor.created_at, Message.id < anchor.id), + ) + ) + skip = 0 + else: + # A bounded window starts just above the match, even in very old threads. + newer = ( + select(func.count()) + .select_from(Message) + .where( + Message.channel_id == channel_id, + Message.parent_id == parent_id, + or_( + Message.created_at > anchor.created_at, + and_(Message.created_at == anchor.created_at, Message.id > anchor.id), + ), + ) + ) + skip = max(0, (await db.scalar(newer)) - 10) result = await db.execute( - select(Message) - .filter_by(channel_id=channel_id, parent_id=parent_id) - .order_by(Message.created_at.desc()) - .offset(skip) - .limit(limit) + stmt.order_by(Message.created_at.desc(), Message.id.desc()).offset(skip).limit(limit) ) all_messages = list(result.scalars().all()) diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index e5a61d7ce5..7017da7c2e 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -120,6 +120,9 @@ def _extract_text_from_binary_response( if not filename or '.' not in filename: ext = mimetypes.guess_extension(content_type) or '' filename = f'download{ext}' + elif not mimetypes.guess_type(filename)[0] and content_type != 'application/octet-stream': + # The dot in names like arXiv's 1706.03762 is not an extension. + filename += mimetypes.guess_extension(content_type) or '' suffix = '.' + filename.split('.')[-1].lower() if '.' in filename else '' @@ -236,7 +239,7 @@ async def get_content_from_url(request, url: str, *, config: dict | None = None) def _get_content_from_url_sync(request, url: str, config: dict): - from open_webui.retrieval.web.utils import validate_url, get_ssrf_safe_requests_session + from open_webui.retrieval.web.utils import WEB_LOADER_HEADERS, validate_url, get_ssrf_safe_requests_session # Validate URL before making any request (blocks private IPs, non-HTTP, filter list) validate_url(url) @@ -261,6 +264,9 @@ def _get_content_from_url_sync(request, url: str, config: dict): try: # Probe through the connect-time SSRF guard; bare requests.get re-resolves (DNS-rebinding gap). session = get_ssrf_safe_requests_session() + # Match the web loader's request, or sites that block unknown clients make a file look like a web page. + session.headers = WEB_LOADER_HEADERS + session.verify = config['web.loader.ssl_verification'] response = session.get(url, stream=True, timeout=30, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS) response.raise_for_status() content_type = response.headers.get('Content-Type', '') diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index e124d4328c..814bf0d886 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -1031,6 +1031,17 @@ class SafePlaywrightURLLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): raise e +WEB_LOADER_HEADERS = { + 'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Referer': 'https://www.google.com/', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', +} + + class SafeWebBaseLoader(BaseLoader): """Fetch pages with connect-time address checks and bounded concurrency.""" @@ -1059,15 +1070,7 @@ class SafeWebBaseLoader(BaseLoader): # Preserve the synchronous loader's environment-proxy behavior. self.session = get_ssrf_safe_requests_session() self.session.verify = verify_ssl - self.session.headers = { - 'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Referer': 'https://www.google.com/', - 'DNT': '1', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - } + self.session.headers = WEB_LOADER_HEADERS async def fetch_all(self, urls): semaphore = asyncio.Semaphore(self.requests_per_second) diff --git a/backend/open_webui/routers/auths.py b/backend/open_webui/routers/auths.py index 9805b32b7a..701acfb360 100644 --- a/backend/open_webui/routers/auths.py +++ b/backend/open_webui/routers/auths.py @@ -82,7 +82,7 @@ from open_webui.utils.json_codec import JSONCodec from open_webui.utils.mfa import MFA_CONFIG_KEYS, MfaConfigForm, get_mfa_config, limit_account, update_mfa_config from open_webui.utils.misc import parse_duration, validate_email_format from open_webui.utils.rate_limit import RateLimiter -from pydantic import BaseModel, StrictStr, field_validator +from pydantic import BaseModel, Field, StrictStr, field_validator from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -128,11 +128,14 @@ ADMIN_CONFIG_KEYS = { 'AUTOMATION_MIN_INTERVAL': 'automations.min_interval', 'ENABLE_AUTOMATIONS': 'automations.enable', 'ENABLE_CHANNELS': 'channels.enable', + 'ENABLE_ENCRYPTED_CHANNELS': 'channels.encryption.enable', 'ENABLE_APPS': 'apps.enable', 'CHANNEL_MODEL_RESPONSE_MODE': 'channels.model_response_mode', 'ENABLE_CALENDAR': 'calendar.enable', 'ENABLE_MEMORIES': 'memories.enable', 'ENABLE_MEMORY_SYSTEM_CONTEXT': 'memories.system_context.enable', + 'SKILL_CONTENT_MAX_CHARS': 'skills.content_max_chars', + 'SKILL_PREVIEW_MAX_CHARS': 'skills.preview_max_chars', 'ENABLE_NOTES': 'notes.enable', 'ENABLE_USER_WEBHOOKS': 'ui.enable_user_webhooks', 'ENABLE_USER_STATUS': 'users.enable_status', @@ -1178,11 +1181,14 @@ class AdminConfig(MfaConfigForm): AUTOMATION_MIN_INTERVAL: int | str | None = None ENABLE_AUTOMATIONS: bool ENABLE_CHANNELS: bool + ENABLE_ENCRYPTED_CHANNELS: bool = False ENABLE_APPS: bool = False CHANNEL_MODEL_RESPONSE_MODE: str = 'thread' ENABLE_CALENDAR: bool ENABLE_MEMORIES: bool ENABLE_MEMORY_SYSTEM_CONTEXT: bool + SKILL_CONTENT_MAX_CHARS: int | None = Field(default=None, gt=0, strict=True) + SKILL_PREVIEW_MAX_CHARS: int | None = Field(default=None, gt=0, strict=True) ENABLE_NOTES: bool ENABLE_USER_WEBHOOKS: bool ENABLE_USER_STATUS: bool @@ -1225,10 +1231,16 @@ async def update_admin_config(request: Request, form_data: AdminConfig, user=Dep updates.pop(key, None) if 'ENABLE_LOGIN_FORM' not in form_data.model_fields_set: updates.pop('ui.enable_login_form', None) + if 'ENABLE_ENCRYPTED_CHANNELS' not in form_data.model_fields_set: + updates.pop('channels.encryption.enable', None) if 'ENABLE_APPS' not in form_data.model_fields_set: updates.pop('apps.enable', None) if 'I18N' not in form_data.model_fields_set: updates.pop('ui.i18n', None) + if 'SKILL_CONTENT_MAX_CHARS' not in form_data.model_fields_set: + updates.pop('skills.content_max_chars', None) + if 'SKILL_PREVIEW_MAX_CHARS' not in form_data.model_fields_set: + updates.pop('skills.preview_max_chars', None) updates['ui.default_interface_settings'] = form_data.DEFAULT_INTERFACE_SETTINGS or {} updates['folders.max_file_count'] = int(form_data.FOLDER_MAX_FILE_COUNT) if form_data.FOLDER_MAX_FILE_COUNT else '' updates['automations.max_count'] = int(form_data.AUTOMATION_MAX_COUNT) if form_data.AUTOMATION_MAX_COUNT else '' diff --git a/backend/open_webui/routers/channels.py b/backend/open_webui/routers/channels.py index 6ef96c8062..805e7fa65b 100644 --- a/backend/open_webui/routers/channels.py +++ b/backend/open_webui/routers/channels.py @@ -42,6 +42,7 @@ from open_webui.models.users import ( Users, ) from open_webui.socket.main import ( + encryption, emit_to_users, enter_room_for_users, get_user_ids_from_room, @@ -657,6 +658,7 @@ async def update_is_active_member_by_id_and_user_id( if not await Channels.is_user_channel_member(channel.id, user.id, db=db): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + await encryption.invalidate(id) await Channels.update_member_active_status(channel.id, user.id, form_data.is_active, db=db) await publish_event( request, @@ -753,6 +755,7 @@ async def remove_members_by_id( raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) try: + await encryption.invalidate(id) deleted = await Channels.remove_members_from_channel(channel.id, form_data.user_ids, db=db) if channel.type == 'group': await leave_room_for_users(f'channel:{channel.id}', form_data.user_ids) @@ -803,6 +806,7 @@ async def update_channel_by_id( previous_access_grants = channel.access_grants try: + await encryption.invalidate(id) channel = await Channels.update_channel_by_id(id, form_data, db=db) # Group and DM channels use membership instead of access grants. if form_data.access_grants is not None and channel.type not in ['group', 'dm']: @@ -847,6 +851,7 @@ async def delete_channel_by_id( raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) try: + await encryption.invalidate(id) await Channels.delete_channel_by_id(id, db=db) await sio.close_room(f'channel:{id}') await publish_event( @@ -1680,6 +1685,8 @@ async def get_channel_thread_messages( message_id: str, skip: int = 0, limit: int = 50, + around: str | None = None, + before: str | None = None, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session), ): @@ -1695,7 +1702,9 @@ async def get_channel_thread_messages( if user.role != 'admin' and not await channel_has_access(user.id, channel, permission='read', db=db): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) - message_list = await Messages.get_messages_by_parent_id(id, message_id, skip, limit, db=db) + message_list = await Messages.get_messages_by_parent_id( + id, message_id, skip, min(max(limit, 1), 100), db=db, around=around, before=before + ) if not message_list: return [] diff --git a/backend/open_webui/routers/chats.py b/backend/open_webui/routers/chats.py index 2a4e8ca033..0ed3f5c738 100644 --- a/backend/open_webui/routers/chats.py +++ b/backend/open_webui/routers/chats.py @@ -864,6 +864,9 @@ async def import_chats( try: chats = await Chats.import_chats(user.id, form_data.chats, db=db) + await Tags.ensure_tags_exist( + list({tag_id for chat in chats for tag_id in (chat.meta or {}).get('tags', [])}), user.id, db=db + ) await publish_event( request, EVENTS.CHAT_IMPORTED, @@ -957,7 +960,7 @@ async def search_user_chats( words = text.strip().split(' ') if page == 1 and len(words) == 1 and words[0].startswith('tag:'): tag_id = words[0].replace('tag:', '') - if len(chat_list) == 0: + if len(chat_list) == 0 and await Chats.count_chats_by_tag_name_and_user_id(tag_id, user.id, db=db) == 0: if await Tags.get_tag_by_name_and_user_id(tag_id, user.id, db=db): log.debug('deleting tag: %s', tag_id) await Tags.delete_tag_by_name_and_user_id(tag_id, user.id, db=db) @@ -1837,6 +1840,13 @@ async def fork_chat_by_id( ) if not is_owner: updated_chat = (await shared_chat_response(chat.model_copy(update={'chat': updated_chat}), user, db=db))['chat'] + messages = list((updated_chat.get('history', {}).get('messages') or {}).values()) + messages.extend(updated_chat.get('messages') or []) + for message in messages: + if (message.get('user_id') or chat.user_id) == chat.user_id: + message['user_id'] = user.id + if message.get('role') == 'user': + message['user'] = {'id': user.id, 'name': user.name} meta = { **((chat.meta or {}) if is_owner else {}), 'forked_from': chat.id, @@ -1979,6 +1989,13 @@ async def clone_shared_chat_by_id( 'branchPointMessageId': chat.chat['history']['currentId'], 'title': f'Clone of {chat.title}', } + messages = list((updated_chat.get('history', {}).get('messages') or {}).values()) + messages.extend(updated_chat.get('messages') or []) + for message in messages: + if (message.get('user_id') or chat.user_id) == chat.user_id: + message['user_id'] = user.id + if message.get('role') == 'user': + message['user'] = {'id': user.id, 'name': user.name} chats = await Chats.import_chats( user.id, diff --git a/backend/open_webui/routers/folders.py b/backend/open_webui/routers/folders.py index f4132222b8..47ee642fd5 100644 --- a/backend/open_webui/routers/folders.py +++ b/backend/open_webui/routers/folders.py @@ -267,19 +267,22 @@ async def get_shared_folders( ] # Also include child folders of shared folders (inheritance) - seen_ids = {folder.id for folder in shared_folders} + result_by_id = {folder['id']: folder for folder in results} for folder in shared_folders: children = await Folders.get_children_folders_by_id_and_user_id(folder.id, folder.user_id, db=db) for child in children or []: - if child.id not in seen_ids: - seen_ids.add(child.id) - results.append( - { - **child.model_dump(), - 'owner_name': owner_names.get(child.user_id, 'Unknown'), - 'permission': 'write' if child.user_id == user.id else folder_perms[folder.id], - } - ) + permission = 'write' if folder.user_id == user.id or child.user_id == user.id else folder_perms[folder.id] + if child.id in result_by_id: + # A weaker direct grant must not hide inherited write access. + if permission == 'write': + result_by_id[child.id]['permission'] = 'write' + else: + result_by_id[child.id] = { + **child.model_dump(), + 'owner_name': owner_names.get(child.user_id, 'Unknown'), + 'permission': permission, + } + results.append(result_by_id[child.id]) return results diff --git a/backend/open_webui/routers/groups.py b/backend/open_webui/routers/groups.py index 8b4be1f3cb..5cbb8840bd 100755 --- a/backend/open_webui/routers/groups.py +++ b/backend/open_webui/routers/groups.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Optional, Literal from fastapi import APIRouter, Depends, HTTPException, Request, status, Query -from open_webui.config import CACHE_DIR +from open_webui.config import CACHE_DIR, DEFAULT_USER_PERMISSIONS from open_webui.models.config import Config from open_webui.constants import ERROR_MESSAGES from open_webui.events import EVENTS, publish_event @@ -33,7 +33,7 @@ from open_webui.utils.auth import get_admin_user, get_verified_user from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, func, or_ from pydantic import BaseModel -from open_webui.utils.access_control import combine_permissions +from open_webui.utils.access_control import combine_permissions, fill_missing_permissions from open_webui.utils.json_codec import JSONCodec log = logging.getLogger(__name__) @@ -418,7 +418,9 @@ async def preview_group_access( active_models = [m for m in all_models if m.is_active] ancestors = (await db.execute(select(Group).where(Group.id.in_(group_ids - {id})))).scalars().all() - inherited_permissions = JSONCodec.loads(JSONCodec.dumps(await Config.get('user.permissions') or {})) + inherited_permissions = fill_missing_permissions( + JSONCodec.loads(JSONCodec.dumps(await Config.get('user.permissions') or {})), DEFAULT_USER_PERMISSIONS + ) for ancestor in ancestors: inherited_permissions = combine_permissions(inherited_permissions, ancestor.permissions or {}) effective_permissions = combine_permissions( diff --git a/backend/open_webui/routers/prompts.py b/backend/open_webui/routers/prompts.py index 967d1b59ea..664fc892a7 100644 --- a/backend/open_webui/routers/prompts.py +++ b/backend/open_webui/routers/prompts.py @@ -7,7 +7,7 @@ from open_webui.config import BYPASS_ADMIN_ACCESS_CONTROL from open_webui.constants import ERROR_MESSAGES from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_session -from open_webui.models.access_grants import AccessGrants +from open_webui.models.access_grants import AccessGrants, normalize_access_grants from open_webui.models.config import Config from open_webui.models.groups import Groups from open_webui.models.prompt_history import ( @@ -289,13 +289,28 @@ async def update_prompt_by_id( detail=ERROR_MESSAGES.COMMAND_TAKEN, ) - form_data.access_grants = await filter_allowed_access_grants( - await Config.get('user.permissions'), - user.id, - user.role, - form_data.access_grants, - 'sharing.public_prompts', - ) + if form_data.access_grants is not None: + existing_access_grants = { + (grant.principal_type, grant.principal_id, grant.permission) for grant in prompt.access_grants + } + submitted_access_grants_map = { + (grant['principal_type'], grant['principal_id'], grant['permission']): grant + for grant in normalize_access_grants(form_data.access_grants) + } + preserved_access_grants = [ + grant for key, grant in submitted_access_grants_map.items() if key in existing_access_grants + ] + new_access_grants = [ + grant for key, grant in submitted_access_grants_map.items() if key not in existing_access_grants + ] + + form_data.access_grants = preserved_access_grants + await filter_allowed_access_grants( + await Config.get('user.permissions'), + user.id, + user.role, + new_access_grants, + 'sharing.public_prompts', + ) # Use the ID from the found prompt updated_prompt = await Prompts.update_prompt_by_id(prompt.id, form_data, user.id, db=db) diff --git a/backend/open_webui/routers/search.py b/backend/open_webui/routers/search.py new file mode 100644 index 0000000000..833b5ae71d --- /dev/null +++ b/backend/open_webui/routers/search.py @@ -0,0 +1,442 @@ +"""Permission-scoped workspace search. Shared snapshots never read the live chat.""" + +from typing import Literal +from urllib.parse import urlencode + +from fastapi import APIRouter, Depends, Query +from open_webui.internal.db import get_async_session +from open_webui.models.access_grants import AccessGrant, AccessGrants +from open_webui.models.channels import ChannelMember, Channels +from open_webui.models.chats import Chat +from open_webui.models.config import Config +from open_webui.models.folders import Folder +from open_webui.models.groups import Groups +from open_webui.models.messages import Message +from open_webui.models.notes import Note, PinnedNote +from open_webui.models.shared_chats import SharedChat +from open_webui.models.users import User +from open_webui.utils.access_control import has_permission +from open_webui.utils.auth import get_verified_user +from pydantic import BaseModel, Field +from sqlalchemy import BigInteger, and_, case, cast, func, literal, or_, select, text, union_all +from sqlalchemy.ext.asyncio import AsyncSession + +router = APIRouter() +PAGE_SIZE = 24 + + +class SearchResult(BaseModel): + id: str + kind: Literal['chat', 'message', 'note'] + title: str + context: str + snippet: str + preview: str + url: str + updated_at: int + shared: bool = False + pinned: bool = False + rank: int = Field(exclude=True, default=3) + + +class SearchScope(BaseModel): + id: str + name: str + kind: Literal['folder', 'channel'] + + +class SearchResponse(BaseModel): + items: list[SearchResult] + has_more: bool + scopes: list[SearchScope] + kinds: list[str] + + +def terms_for(query: str) -> list[str]: + return list(dict.fromkeys(query.lower().split()))[:12] + + +def pattern(value: str) -> str: + return '%' + value.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + '%' + + +def excerpt(content: str, query: str, length: int = 200) -> str: + content = content or '' + positions = [content.lower().find(term) for term in terms_for(query)] + position = min((p for p in positions if p >= 0), default=0) + start = max(0, position - length // 4) + return ( + ('…' if start else '') + + content[start : start + length].strip() + + ('…' if start + length < len(content) else '') + ) + + +def chat_passage(data: dict, query: str) -> tuple[str, str | None]: + history = (data.get('history') or {}).get('messages') or {} + messages = list(history.values()) if history else data.get('messages') or [] + messages = [m for m in messages if isinstance(m, dict) and isinstance(m.get('content'), str)] + terms = terms_for(query) + match = next((m for m in messages if terms and all(t in m['content'].lower() for t in terms)), None) + if match is None: + match = next((m for m in messages if any(t in m['content'].lower() for t in terms)), None) + fallback = messages[-1] if messages else {} + return (match or fallback).get('content', ''), match.get('id') if match else None + + +def title_rank(title, query: str): + if not query: + return literal(3) + return case((func.lower(title) == query.lower(), 0), (title.ilike(pattern(query), escape='\\'), 1), else_=3) + + +def message_match_sql(table: str, dialect: str, key: str): + # Identifiers are internal constants; all user input remains bound parameters. + if dialect == 'sqlite': + sources = [f"json_each({table}.chat, '$.history.messages')", f"json_each({table}.chat, '$.messages')"] + return text( + '(' + + ' OR '.join( + f"EXISTS (SELECT 1 FROM {source} AS m WHERE json_type(m.value, '$.content') = 'text' " + f"AND lower(m.value ->> 'content') LIKE :{key} ESCAPE '\\')" + for source in sources + ) + + ')' + ) + sources = [f"json_each({table}.chat->'history'->'messages')", f"json_array_elements({table}.chat->'messages')"] + return text( + '(' + + ' OR '.join( + f"EXISTS (SELECT 1 FROM {source} AS m WHERE json_typeof(m.value->'content') = 'string' " + f"AND lower(m.value->>'content') LIKE :{key} ESCAPE '\\')" + for source in sources + ) + + ')' + ) + + +async def search_workspace( # noqa: C901 — keep permission scoping and source dispatch together + db: AsyncSession, + user, + query: str = '', + kind: str = 'all', + sharing: str = 'all', + scope: str = '', + page: int = 1, + enabled: tuple[str, ...] = ('chat', 'message', 'note'), +) -> SearchResponse: + groups = [g.id for g in await Groups.get_groups_by_member_id(user.id, db=db, include_inherited=True)] + principal = or_( + and_(AccessGrant.principal_type == 'user', AccessGrant.principal_id == user.id), + and_(AccessGrant.principal_type == 'group', AccessGrant.principal_id.in_(groups)), + ) + + def granted(resource, resource_id, public=False): + principals = ( + or_(principal, and_(AccessGrant.principal_type == 'user', AccessGrant.principal_id == '*')) + if public + else principal + ) + return ( + select(AccessGrant.id) + .where( + AccessGrant.resource_type == resource, + AccessGrant.resource_id == resource_id, + AccessGrant.permission == 'read', + principals, + ) + .exists() + ) + + # UNION (not UNION ALL) terminates even if malformed folder data contains a cycle. + folders = ( + select(Folder.id) + .where(or_(Folder.user_id == user.id, granted('folder', Folder.id, True))) + .cte('search_folders', recursive=True) + ) + folders = folders.union(select(Folder.id).join(folders, Folder.parent_id == folders.c.id)) + folder_rows = ( + await db.execute( + select(Folder.id, Folder.name, Folder.parent_id, Folder.user_id).where(Folder.id.in_(select(folders.c.id))) + ) + ).all() + folder_map = {f.id: f for f in folder_rows} + channels = await Channels.get_channels_by_user_id(user.id, db=db) if 'message' in enabled else [] + channel_map = {c.id: c for c in channels} + channel_names = {c.id: c.name for c in channels} + direct_message_ids = [c.id for c in channels if c.type == 'dm' and not c.name] + if direct_message_ids: + members = ( + await db.execute( + select(ChannelMember.channel_id, User.name) + .join(User, User.id == ChannelMember.user_id) + .where(ChannelMember.channel_id.in_(direct_message_ids), User.id != user.id) + .order_by(User.name, User.id) + ) + ).all() + participants = {channel_id: [] for channel_id in direct_message_ids} + for channel_id, name in members: + participants[channel_id].append(name) + for channel_id, names in participants.items(): + channel_names[channel_id] = ', '.join(names) or user.name + scopes = [SearchScope(id='folder:' + f.id, name=f.name, kind='folder') for f in folder_rows] + scopes += [SearchScope(id='channel:' + c.id, name=channel_names[c.id], kind='channel') for c in channels] + scopes.sort(key=lambda s: (s.kind, s.name.lower(), s.id)) + shared_folder_ids = set( + ( + await db.execute( + select(Folder.id).where( + Folder.id.in_(folder_map), + select(AccessGrant.id) + .where( + AccessGrant.resource_type == 'folder', + AccessGrant.resource_id == Folder.id, + AccessGrant.permission == 'read', + ) + .exists(), + ) + ) + ).scalars() + ) + while True: + children = {f.id for f in folder_rows if f.parent_id in shared_folder_ids} + if children.issubset(shared_folder_ids): + break + shared_folder_ids |= children + scoped_folders = set(folder_map) + if scope.startswith('folder:'): + root = scope.removeprefix('folder:') + scoped_folders = {root} if root in folder_map else set() + while True: + children = {f.id for f in folder_rows if f.parent_id in scoped_folders} + if children.issubset(scoped_folders): + break + scoped_folders |= children + if scope and scope not in {s.id for s in scopes}: + return SearchResponse(items=[], has_more=False, scopes=scopes, kinds=list(enabled)) + + query = query.strip() + terms = terms_for(query) + results = [] + dialect = db.bind.dialect.name + + def sharing_filter(owner): + if sharing == 'mine': + return owner == user.id + if sharing == 'shared': + return owner != user.id + return True + + def apply_match(stmt, model, body=None): + params = {} + for index, term in enumerate(terms): + key = f'search_{index}' + params[key] = pattern(term) + content_match = ( + body.ilike(pattern(term), escape='\\') + if body is not None + else message_match_sql(model.__tablename__, dialect, key) + ) + stmt = stmt.where(or_(model.title.ilike(pattern(term), escape='\\'), content_match)) + return stmt.params(**params) + + def append_result(id, result_kind, title, context, content, url, updated, shared, pinned=False, rank=3): + results.append( + SearchResult( + id=id, + kind=result_kind, + title=title or 'Untitled', + context=context, + snippet=excerpt(content, query), + preview=excerpt(content, query, 2400), + url=url, + updated_at=updated, + shared=shared, + pinned=bool(pinned), + rank=rank, + ) + ) + + sources = {} + if 'chat' in enabled and kind in ('all', 'chat') and not scope.startswith('channel:'): + continue_shared = select(SharedChat.chat_id).where( + SharedChat.chat['share_mode'].as_string() == 'continue', granted('shared_chat', SharedChat.chat_id) + ) + live_access = or_( + Chat.user_id == user.id, Chat.folder_id.in_(select(folders.c.id)), Chat.id.in_(continue_shared) + ) + live_ids = select(Chat.id).where(live_access) + rank = ( + title_rank(Chat.title, query) + if query + else case((and_(Chat.pinned.is_(True), Chat.user_id == user.id), 0), else_=3) + ) + stmt = select(Chat).where( + live_access, + Chat.archived.is_(False), + func.coalesce(Chat.meta['internal'].as_boolean(), False).is_(False), + sharing_filter(Chat.user_id), + ) + if scope: + stmt = stmt.where(Chat.folder_id.in_(scoped_folders)) + sources['chat'] = (Chat, apply_match(stmt, Chat), rank, Chat.updated_at) + if not scope and sharing != 'mine': + stmt = select(SharedChat).where( + granted('shared_chat', SharedChat.chat_id), ~SharedChat.chat_id.in_(live_ids) + ) + if sharing == 'shared': + stmt = stmt.where(SharedChat.user_id != user.id) + sources['shared'] = ( + SharedChat, + apply_match(stmt, SharedChat), + title_rank(SharedChat.title, query), + SharedChat.updated_at, + ) + + if 'note' in enabled and kind in ('all', 'note') and not scope: + pinned = select(PinnedNote.id).where(PinnedNote.note_id == Note.id, PinnedNote.user_id == user.id).exists() + note_shared = ( + select(AccessGrant.id) + .where( + AccessGrant.resource_type == 'note', + AccessGrant.resource_id == Note.id, + AccessGrant.permission == 'read', + ) + .exists() + ) + stmt = AccessGrants.has_permission_filter( + db, select(Note, pinned, note_shared), Note, {'user_id': user.id, 'group_ids': groups}, 'note', 'read' + ) + stmt = stmt.where(sharing_filter(Note.user_id)) + stmt = apply_match(stmt, Note, Note.data['content']['md'].as_string()) + rank = title_rank(Note.title, query) if query else case((pinned, 0), else_=3) + sources['note'] = (Note, stmt, rank, cast(Note.updated_at.op('/')(literal(1_000_000_000)), BigInteger)) + + if 'message' in enabled and kind in ('all', 'message') and not scope.startswith('folder:'): + ids = [scope.removeprefix('channel:')] if scope else list(channel_map) + stmt = select(Message, User.name).outerjoin(User, User.id == Message.user_id).where(Message.channel_id.in_(ids)) + stmt = stmt.where(sharing_filter(Message.user_id)) + for term in terms: + stmt = stmt.where(Message.content.ilike(pattern(term), escape='\\')) + sources['message'] = ( + Message, + stmt, + literal(3), + cast(Message.updated_at.op('/')(literal(1_000_000_000)), BigInteger), + ) + + if not sources: + return SearchResponse(items=[], has_more=False, scopes=scopes, kinds=list(enabled)) + + # Page one combined, permission-filtered ID query. Only hydrate the 24 visible + # results, rather than repeatedly loading full conversation JSON while scrolling. + candidates = union_all( + *[ + stmt.with_only_columns( + literal(source).label('source'), + model.id.label('id'), + rank.label('rank'), + updated.label('updated_at'), + maintain_column_froms=True, + ) + for source, (model, stmt, rank, updated) in sources.items() + ] + ).subquery() + matches = ( + await db.execute( + select(candidates) + .order_by(candidates.c.rank, candidates.c.updated_at.desc(), candidates.c.source, candidates.c.id) + .offset((page - 1) * PAGE_SIZE) + .limit(PAGE_SIZE + 1) + ) + ).all() + has_more = len(matches) > PAGE_SIZE + matches = matches[:PAGE_SIZE] + hydrated = {} + for source, (model, stmt, _, _) in sources.items(): + ids = [match.id for match in matches if match.source == source] + if ids: + hydrated[source] = {row[0].id: row for row in (await db.execute(stmt.where(model.id.in_(ids)))).all()} + + for match in matches: + row = hydrated.get(match.source, {}).get(match.id) + if row is None: + continue # Access may have been revoked between selecting IDs and hydrating. + item = row[0] + if match.source in ('chat', 'shared'): + content, message_id = chat_passage(item.chat or {}, query) + snapshot = match.source == 'shared' + folder = None if snapshot else folder_map.get(item.folder_id) + context = ( + 'Shared snapshot' + if snapshot + else (folder.name if folder else ('Shared chat' if item.user_id != user.id else 'Personal chat')) + ) + url = ('/s/' if snapshot else '/c/') + item.id + if message_id: + url += '?' + urlencode({'message': message_id}) + is_shared = ( + snapshot or item.user_id != user.id or bool(item.share_id) or item.folder_id in shared_folder_ids + ) + append_result( + match.source + ':' + item.id, + 'chat', + item.title, + context, + content, + url, + match.updated_at, + is_shared, + not snapshot and item.pinned and item.user_id == user.id, + match.rank, + ) + elif match.source == 'note': + content = ((item.data or {}).get('content') or {}).get('md') or '' + append_result( + 'note:' + item.id, + 'note', + item.title, + 'Shared note' if item.user_id != user.id or row[2] else 'Personal note', + content if isinstance(content, str) else '', + '/notes/' + item.id + ('?' + urlencode({'search': query}) if query else ''), + match.updated_at, + item.user_id != user.id or row[2], + row[1], + match.rank, + ) + else: + channel = channel_map[item.channel_id] + url = '/channels/' + channel.id + '?' + urlencode({'thread': item.parent_id or item.id, 'message': item.id}) + append_result( + 'message:' + item.id, + 'message', + row[1] or 'Message', + channel_names[channel.id] if channel.type == 'dm' else '#' + channel_names[channel.id], + item.content, + url, + match.updated_at, + True, + rank=match.rank, + ) + + return SearchResponse(items=results, has_more=has_more, scopes=scopes, kinds=list(enabled)) + + +@router.get('', response_model=SearchResponse) +async def search( + q: str = Query('', max_length=500), + kind: Literal['all', 'chat', 'message', 'note'] = 'all', + sharing: Literal['all', 'mine', 'shared'] = 'all', + scope: str = Query('', max_length=200), + page: int = Query(1, ge=1, le=100_000), + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + enabled = ['chat'] + permissions = await Config.get('user.permissions') + for feature, result_kind in [('channels', 'message'), ('notes', 'note')]: + if await Config.get(feature + '.enable') and ( + user.role == 'admin' or await has_permission(user.id, 'features.' + feature, permissions, db=db) + ): + enabled.append(result_kind) + return await search_workspace(db, user, q, kind, sharing, scope, page, tuple(enabled)) diff --git a/backend/open_webui/routers/tools.py b/backend/open_webui/routers/tools.py index 383d6669b1..19bc3fc67e 100644 --- a/backend/open_webui/routers/tools.py +++ b/backend/open_webui/routers/tools.py @@ -19,7 +19,7 @@ from open_webui.env import ( ) from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_session -from open_webui.models.access_grants import AccessGrants +from open_webui.models.access_grants import AccessGrants, normalize_access_grants from open_webui.models.config import Config from open_webui.models.groups import Groups from open_webui.models.oauth_sessions import OAuthSessions @@ -571,12 +571,26 @@ async def _update_tool(request, id, form_data, user, db, version_id=None): specs = get_local_tool_specs(tool_module) - if version_id is None: - form_data.access_grants = await filter_allowed_access_grants( + if version_id is None and form_data.access_grants is not None: + existing_access_grants = { + (grant.principal_type, grant.principal_id, grant.permission) for grant in tools.access_grants + } + submitted_access_grants_map = { + (grant['principal_type'], grant['principal_id'], grant['permission']): grant + for grant in normalize_access_grants(form_data.access_grants) + } + preserved_access_grants = [ + grant for key, grant in submitted_access_grants_map.items() if key in existing_access_grants + ] + new_access_grants = [ + grant for key, grant in submitted_access_grants_map.items() if key not in existing_access_grants + ] + + form_data.access_grants = preserved_access_grants + await filter_allowed_access_grants( await Config.get('user.permissions'), user.id, user.role, - form_data.access_grants, + new_access_grants, 'sharing.public_tools', ) diff --git a/backend/open_webui/routers/users.py b/backend/open_webui/routers/users.py index 69e634a8c5..90a327e3fd 100644 --- a/backend/open_webui/routers/users.py +++ b/backend/open_webui/routers/users.py @@ -5,11 +5,13 @@ import io import logging import time from collections import Counter +from copy import deepcopy from datetime import datetime, timedelta from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from fastapi.responses import FileResponse, Response, StreamingResponse +from open_webui.config import DEFAULT_USER_PERMISSIONS from open_webui.constants import ERROR_MESSAGES from open_webui.events import EVENTS, publish_event from open_webui.env import ENABLE_PROFILE_IMAGE_URL_FORWARDING, PROFILE_IMAGE_ALLOWED_MIME_TYPES, STATIC_DIR @@ -37,7 +39,7 @@ from open_webui.models.access_grants import AccessGrants from open_webui.models.knowledge import Knowledges from open_webui.models.models import Models from open_webui.models.tools import Tools -from open_webui.utils.access_control import get_permissions, has_permission +from open_webui.utils.access_control import fill_missing_permissions, get_permissions, has_permission from open_webui.utils.auth import ( get_admin_user, get_password_hash, @@ -262,6 +264,7 @@ class FeaturesPermissions(BaseModel): api_keys: bool = False notes: bool = True channels: bool = True + encrypted_channels: bool = False folders: bool = True direct_tool_servers: bool = False @@ -412,7 +415,9 @@ def _calculate_streaks(heatmap: list[dict]) -> dict[str, int]: @router.get('/default/permissions', response_model=UserPermissions) async def get_default_user_permissions(request: Request, user=Depends(get_admin_user)): - user_permissions = await Config.get('user.permissions') + user_permissions = fill_missing_permissions( + deepcopy(await Config.get('user.permissions') or {}), DEFAULT_USER_PERMISSIONS + ) return { 'workspace': WorkspacePermissions(**user_permissions.get('workspace', {})), 'sharing': SharingPermissions(**user_permissions.get('sharing', {})), diff --git a/backend/open_webui/socket/encryption.py b/backend/open_webui/socket/encryption.py new file mode 100644 index 0000000000..227fb9556d --- /dev/null +++ b/backend/open_webui/socket/encryption.py @@ -0,0 +1,432 @@ +"""Volatile packet relay. Only account bindings and liveness enter coordination storage.""" + +import asyncio +import base64 +import hashlib +import json +import time +from contextlib import asynccontextmanager +from contextvars import ContextVar +from copy import deepcopy +from uuid import uuid4 + +from open_webui.config import DEFAULT_USER_PERMISSIONS +from open_webui.env import ( + ENABLE_WEBSOCKET_SUPPORT, + ENV, + REDIS_KEY_PREFIX, + WEBSOCKET_SERVER_ENGINEIO_LOGGING, + WEBSOCKET_SERVER_LOGGING, +) +from open_webui.models.channels import Channels +from open_webui.models.config import Config +from open_webui.models.groups import Groups +from open_webui.models.users import Users +from open_webui.utils.access_control import combine_permissions, fill_missing_permissions + +MAX_PACKET = 1024 * 1024 +TTL = 90 + + +class Rejected(Exception): + pass + + +def require(condition): + if not condition: + raise Rejected() + + +def packet(value): + require(isinstance(value, str) and 0 < len(value) <= (MAX_PACKET + 2) // 3 * 4) + try: + decoded = base64.b64decode(value, validate=True) + except (ValueError, TypeError): + raise Rejected() from None + require(0 < len(decoded) <= MAX_PACKET) + return value + + +class Registry: + def __init__(self, redis=None): + self.redis = redis + self.values = {} + self.lock = asyncio.Lock() + self.lease = ContextVar('encryption_lease', default=None) + + def key(self, channel): + digest = hashlib.sha256(channel.encode()).hexdigest() + return f'{REDIS_KEY_PREFIX}:encryption:{{{digest}}}' + + @asynccontextmanager + async def transaction(self, channel): + key = self.key(channel) + lock = self.redis.lock(key + ':lock', timeout=10, blocking_timeout=2) if self.redis else self.lock + async with lock: + raw = await self.redis.get(key) if self.redis else self.values.get(key) + state = json.loads(raw) if raw else None + if state and state['expires'] <= time.time(): + state = None + token = self.lease.set(lock if self.redis else None) + try: + yield state + finally: + self.lease.reset(token) + + async def save(self, channel, state): + key = self.key(channel) + if state is not None: + state['expires'] = time.time() + TTL + raw = json.dumps(state, separators=(',', ':')) if state is not None else '' + if self.redis: + lease = self.lease.get() + require(lease is not None) + # A stalled worker must never overwrite a newer owner's metadata. + changed = await self.redis.eval( + "if redis.call('get', KEYS[2]) ~= ARGV[1] then return 0 end " + "if ARGV[2] == '' then redis.call('del', KEYS[1]) " + "else redis.call('set', KEYS[1], ARGV[2], 'EX', ARGV[3]) end return 1", + 2, + key, + key + ':lock', + lease.local.token, + raw, + TTL, + ) + if not changed: + raise RuntimeError('Coordination lease expired') + elif state is None: + self.values.pop(key, None) + else: + self.values[key] = raw + + +class Relay: + def __init__(self, sio, authenticate, redis=None): + self.sio, self.authenticate = sio, authenticate + self.registry = Registry(redis) + self.local = {} # sid -> channel, never payloads + self.rates = {} + for action in ('start', 'status', 'join', 'bind', 'packet', 'leave', 'ping'): + + async def handler(sid, data=None, action=action): + return await self.handle(action, sid, data) + + sio.on(f'encryption:{action}', handler) + + async def authorize(self, sid, channel): + require(ENABLE_WEBSOCKET_SUPPORT and self.sio.transport(sid) == 'websocket') + require(not WEBSOCKET_SERVER_LOGGING and not WEBSOCKET_SERVER_ENGINEIO_LOGGING) + scope = (self.sio.get_environ(sid) or {}).get('asgi.scope', {}) + require(ENV != 'prod' or scope.get('scheme') in ('https', 'wss')) + auth = await self.authenticate(sid, wait_for_disconnect=False) + require(auth and auth.get('id')) + user = await Users.get_user_by_id(auth['id']) + require(user and user.role in ('user', 'admin')) + record = await Channels.get_channel_by_id(channel) + require(record and (record.type == 'dm' or (record.type == 'group' and record.is_private is True))) + require(await Channels.is_user_channel_member(channel, user.id)) + return user + + @staticmethod + def view(state): + if not state: + return None + return { + 'channel': state['channel'], + 'session': state['session'], + 'host': state['host'], + 'bindings': [ + { + 'identity': { + 'channel': state['channel'], + 'session': state['session'], + 'user': p['user'], + 'participation': pid, + }, + 'key': p['key'], + } + for pid, p in state['members'].items() + if p.get('key') + ], + 'participants': [ + {'participation': pid, 'user': p['user'], 'name': p['name']} for pid, p in state['members'].items() + ], + } + + async def invite(self, state, channel): + await self.sio.emit( + 'encryption:available', + {'channel': channel, 'session': state['session'] if state else None}, + room=f'channel:{channel}', + ) + + async def deliver(self, state, recipients, event): + # Per-recipient sequence detects missing Pub/Sub deliveries without storing packets. + targets = [] + for pid in recipients: + member = state['members'][pid] + member['out'] += 1 + targets.append((member['sid'], member['out'])) + await self.registry.save(state['channel'], state) + for sid, sequence in targets: + await self.sio.emit('encryption:event', {**event, **self.view(state), 'sequence': sequence}, to=sid) + + async def close(self, channel, state): + await self.registry.save(channel, None) + if state: + for p in state['members'].values(): + self.local.pop(p['sid'], None) + await self.sio.emit( + 'encryption:event', {'kind': 'ended', 'channel': channel, 'session': state['session']}, to=p['sid'] + ) + await self.invite(None, channel) + + async def eligible_users(self, user_ids): + config = await Config.get_many('channels.enable', 'channels.encryption.enable', 'user.permissions') + if not config['channels.enable'] or not config['channels.encryption.enable']: + return {} + defaults = fill_missing_permissions(deepcopy(config['user.permissions'] or {}), DEFAULT_USER_PERMISSIONS) + users = await Users.get_users_by_user_ids(list(set(user_ids))) + groups = await Groups.get_groups_by_member_ids( + [u.id for u in users if u.role == 'user'], include_inherited=True + ) + eligible = {} + for user in users: + if user.role == 'admin': + eligible[user.id] = user + elif user.role == 'user': + permissions = deepcopy(defaults) + for group in groups[user.id]: + combine_permissions(permissions, group.permissions or {}) + features = permissions.get('features', {}) + if features.get('channels') and features.get('encrypted_channels'): + eligible[user.id] = user + return eligible + + async def prune(self, state, eligible=None): + if not state: + return None + if eligible is None: + eligible = await self.eligible_users([p['user'] for p in state['members'].values()]) + members = await Channels.get_members_by_channel_id(state['channel']) + active = {m.user_id for m in members if m.is_active} + stale = [ + pid + for pid, p in state['members'].items() + if p['seen'] < time.time() - 45 or p['user'] not in active or p['user'] not in eligible + ] + if state['host'] in stale: + await self.close(state['channel'], state) + return None + if stale: + await self.remove(state, *stale) + return state + + async def remove(self, state, *pids): + if state['host'] in pids: + await self.close(state['channel'], state) + return + # Exclude every revoked recipient before any departure event is forwarded. + removed = [(pid, state['members'].pop(pid)) for pid in pids] + for pid, p in removed: + self.local.pop(p['sid'], None) + await self.sio.emit( + 'encryption:event', + {'kind': 'ended', 'channel': state['channel'], 'session': state['session']}, + to=p['sid'], + ) + await self.deliver( + state, + [i for i, m in state['members'].items() if m['admitted']], + {'kind': 'departed', 'participation': pid, 'admitted': p['admitted']}, + ) + + async def limit(self, user, action, size): + now = time.monotonic() + if len(self.rates) > 10000: + self.rates = {k: v for k, v in self.rates.items() if now - v[0] < 60} + if self.registry.redis: + bucket = 'admission' if action in ('start', 'join') else 'traffic' + period, maximum = (60, 10) if bucket == 'admission' else (1, 100) + key = self.registry.key('rate:' + user + ':' + bucket) + allowed = await self.registry.redis.eval( + "local n = redis.call('hincrby', KEYS[1], 'count', 1) " + "local b = redis.call('hincrby', KEYS[1], 'bytes', ARGV[1]) " + "if n == 1 then redis.call('expire', KEYS[1], ARGV[2]) end " + 'if n > tonumber(ARGV[3]) or b > tonumber(ARGV[4]) then return 0 end return 1', + 1, + key, + size, + period, + maximum, + 4 * MAX_PACKET, + ) + require(allowed) + return + # Account-wide admission bound also applies across tabs on this worker. + key = (user, 'admission' if action in ('start', 'join') else 'traffic') + period, maximum = (60, 10) if key[1] == 'admission' else (1, 100) + start, count, total = self.rates.get(key, (now, 0, 0)) + if now - start >= period: + start, count, total = now, 0, 0 + require(count < maximum and total + size <= 4 * MAX_PACKET) + self.rates[key] = (start, count + 1, total + size) + + async def reserve(self, action, sid, user, channel, data, state): + # Another worker may have closed the session; its metadata is authoritative. + if self.local.get(sid) == channel and ( + not state or not any(p['sid'] == sid for p in state['members'].values()) + ): + self.local.pop(sid, None) + require(sid not in self.local) + if action == 'start': + require(state is None) + pid = str(uuid4()) + state = {'channel': channel, 'session': str(uuid4()), 'host': pid, 'members': {}} + else: + require(state and data.get('session') == state['session']) + require(len(state['members']) < 128 and not any(p['user'] == user.id for p in state['members'].values())) + require(state['members'][state['host']].get('key')) + pid = str(uuid4()) + state['members'][pid] = { + 'sid': sid, + 'user': user.id, + 'name': user.name, + 'key': None, + 'seen': time.time(), + 'out': 0, + 'admitted': action == 'start', + } + self.local[sid] = channel + await self.registry.save(channel, state) + return {'ok': True, **self.view(state), 'participation': pid} + + async def bind(self, state, pid, member, data): + require(member['key'] is None) + key = packet(data.get('key')) + require(len(base64.b64decode(key)) == 32) + member['key'] = key + if pid != state['host']: + kp = packet(data.get('packet')) + await self.deliver(state, [state['host']], {'kind': 'candidate', 'sender': pid, 'packet': kp}) + else: + await self.registry.save(state['channel'], state) + await self.invite(state, state['channel']) + + async def forward(self, state, pid, member, data): + require(member['key'] and member['admitted']) + body = packet(data.get('packet')) + if data.get('kind') == 'admit': + require(pid == state['host']) + target = data.get('target') + joining = state['members'].get(target) + require(joining and joining.get('key') and not joining['admitted']) + welcome = packet(data.get('welcome')) + existing = [i for i, p in state['members'].items() if i != pid and p['admitted']] + joining['admitted'] = True + await self.deliver(state, existing, {'kind': 'mls', 'sender': pid, 'packet': body}) + await self.deliver(state, [target], {'kind': 'welcome', 'sender': pid, 'packet': welcome}) + else: + require(data.get('kind') == 'mls') + recipients = [i for i, p in state['members'].items() if i != pid and p['admitted']] + await self.deliver(state, recipients, {'kind': 'mls', 'sender': pid, 'packet': body}) + + async def handle(self, action, sid, data): + channel = None + authorized = False + try: + require(isinstance(data, dict)) + channel = data.get('channel') + require(isinstance(channel, str) and 0 < len(channel) <= 128) + size = len(json.dumps(data)) + require(size <= 3 * MAX_PACKET) + user = await self.authorize(sid, channel) + authorized = True + await self.limit(user.id, action, size) + async with self.registry.transaction(channel) as state: + # Read current policy after waiting for the channel's coordination lock. + eligible = await self.eligible_users( + [user.id, *([p['user'] for p in state['members'].values()] if state else [])] + ) + state = await self.prune(state, eligible) + require(user.id in eligible) + user = eligible[user.id] + require(await Channels.is_user_channel_member(channel, user.id)) + if action == 'status': + return { + 'ok': True, + 'available': bool(state and state['members'][state['host']].get('key')), + 'session': state['session'] if state else None, + } + if action in ('start', 'join'): + return await self.reserve(action, sid, user, channel, data, state) + require(state and data.get('session') == state['session'] and self.local.get(sid) == channel) + pid = data.get('participation') + member = state['members'].get(pid) + require(member and member['sid'] == sid and member['user'] == user.id) + member['seen'] = time.time() + if action == 'leave': + await self.remove(state, pid) + return {'ok': True} + if action == 'bind': + await self.bind(state, pid, member, data) + elif action == 'ping': + await self.registry.save(channel, state) + elif action == 'packet': + await self.forward(state, pid, member, data) + else: + raise Rejected() + return {'ok': True, **self.view(state)} + except Rejected: + return {'ok': False, 'error': 'Private chat is unavailable or the session changed.'} + except Exception: + # Never log exceptions carrying packet input. Fail closed on coordinator failure. + if authorized: + await self.fail_local(channel) + return {'ok': False, 'error': 'Private chat ended.'} + + async def fail_local(self, channel): + for sid, local_channel in list(self.local.items()): + if channel is None or local_channel == channel: + self.local.pop(sid, None) + await self.sio.emit( + 'encryption:event', {'kind': 'ended', 'channel': local_channel}, to=sid, ignore_queue=True + ) + + async def disconnect(self, sid): + channel = self.local.pop(sid, None) + if not channel: + return + try: + async with self.registry.transaction(channel) as state: + if state: + for pid, p in list(state['members'].items()): + if p['sid'] == sid: + await self.remove(state, pid) + break + except Exception: + await self.fail_local(channel) + + async def invalidate(self, channel): + # Channel access mutations end the session before any further private sends. + try: + async with self.registry.transaction(channel) as state: + await self.close(channel, state) + except Exception: + await self.fail_local(channel) + + async def sweep(self): + for channel in set(self.local.values()): + try: + async with self.registry.transaction(channel) as state: + if not state: + await self.fail_local(channel) + else: + await self.prune(state) + except Exception: + await self.fail_local(channel) + if not self.registry.redis: + for key, raw in list(self.registry.values.items()): + if json.loads(raw)['expires'] <= time.time(): + self.registry.values.pop(key, None) diff --git a/backend/open_webui/socket/main.py b/backend/open_webui/socket/main.py index 5b2a826fee..09937e9171 100644 --- a/backend/open_webui/socket/main.py +++ b/backend/open_webui/socket/main.py @@ -42,6 +42,7 @@ from open_webui.models.config import Config from open_webui.models.folders import Folders from open_webui.models.notes import Notes, NoteUpdateForm from open_webui.models.users import UserNameResponse, Users +from open_webui.socket.encryption import Relay from open_webui.socket.redis_room_channels import AsyncRedisRoomChannelManager from open_webui.socket.utils import SOCKET_EVENT_LOCKS, CachedRedisDict, RedisDict, RedisLock, YdocManager from open_webui.tasks import ( @@ -231,6 +232,47 @@ def get_session_pool_batches(): return [list(SESSION_POOL.items())] +async def publish_chat_presence(chat_id: str): + # Room maps only contain this worker's sockets; the session pool spans workers. + lock = ( + REDIS.lock( + f'{REDIS_KEY_PREFIX}:chat:{chat_id}:presence:lock', + timeout=WEBSOCKET_REDIS_LOCK_TIMEOUT, + blocking_timeout=WEBSOCKET_REDIS_LOCK_TIMEOUT, + ) + if REDIS is not None + else SOCKET_EVENT_LOCKS.setdefault(('chat', chat_id), asyncio.Lock()) + ) + try: + async with lock, asyncio.timeout(WEBSOCKET_REDIS_LOCK_TIMEOUT / 2): + now = time.time() + people = {} + for batch in get_session_pool_batches(): + for _, session in batch: + if ( + session + and chat_id in (session.get('chat_ids') or []) + and now - session.get('last_seen_at', 0) <= SESSION_POOL_TIMEOUT + ): + people[session['id']] = {'id': session['id'], 'name': session.get('name', '')} + if REDIS is not None and not await lock.owned(): + return + await sio.emit( + 'events', + { + 'chat_id': chat_id, + 'shared': True, + 'data': { + 'type': 'chat:presence', + 'data': {'users': sorted(people.values(), key=lambda person: person['id'])}, + }, + }, + room=f'chat:{chat_id}', + ) + except Exception: + log.exception('Failed to publish presence for chat %s', chat_id) + + async def periodic_session_pool_cleanup(): """Reap orphaned SESSION_POOL entries that missed heartbeats (e.g. crashed instance).""" retry_delay = random.uniform(WEBSOCKET_REDIS_LOCK_TIMEOUT / 2, WEBSOCKET_REDIS_LOCK_TIMEOUT) @@ -257,11 +299,23 @@ async def periodic_session_pool_cleanup(): ] if expired: log.warning('Reaping %d orphaned session(s) from the session pool', len(expired)) + chat_ids = { + chat_id + for sid, entry in batch + if sid in expired + for chat_id in entry.get('chat_ids', []) + } if WEBSOCKET_MANAGER == 'redis': SESSION_POOL.delete_many(*expired) else: for sid in expired: SESSION_POOL.pop(sid, None) + for sid, entry in batch: + if sid in expired: + for chat_id in entry.get('chat_ids', []): + await sio.leave_room(sid, f'chat:{chat_id}') + for chat_id in chat_ids: + await publish_chat_presence(chat_id) await asyncio.sleep(0) # don't hold the loop for the whole sweep next_cleanup_at = time.monotonic() + SESSION_POOL_TIMEOUT @@ -355,6 +409,7 @@ LOCAL_AUTHENTICATED_SIDS: set[str] = set() async def periodic_socket_authentication(): while True: await asyncio.sleep(30) + await encryption.sweep() for sid in tuple(LOCAL_AUTHENTICATED_SIDS): await get_socket_session_user(sid) @@ -376,6 +431,9 @@ async def get_socket_session_user(sid: str, *, wait_for_disconnect: bool = True) return None +encryption = Relay(sio, get_socket_session_user, REDIS) + + def get_session_ids_from_room(room): """Get all session IDs from a specific room.""" members = get_room_sid_map(sio.manager, '/', room) @@ -562,12 +620,12 @@ async def user_join(sid, data): async def refresh_chat_access(chat_id=None): # The pool includes sessions on other workers; Socket.IO routes room changes via Redis. access = {} + chat_ids = set() for batch in get_session_pool_batches(): for sid, session in batch: if not session: continue - chat_ids = set(session.get('chat_ids') or []) - for cid in list(chat_ids): + for cid in session.get('chat_ids') or []: if chat_id and cid != chat_id: continue key = (cid, session['id']) @@ -576,12 +634,19 @@ async def refresh_chat_access(chat_id=None): access[key] = bool(user and await Chats.get_accessible_chat_by_id(cid, user)) if not access[key]: await sio.leave_room(sid, f'chat:{cid}') - chat_ids.discard(cid) + chat_ids.add(cid) + # Preserve joins/leaves and heartbeats that happened during access checks. + current = SESSION_POOL.get(sid) + if current: + SESSION_POOL[sid] = { + **current, + 'chat_ids': [id for id in current.get('chat_ids', []) if id != cid], + } await sio.emit( 'events', {'chat_id': cid, 'shared': True, 'data': {'type': 'chat:access', 'data': {}}}, to=sid ) - if chat_ids != set(session.get('chat_ids') or []): - SESSION_POOL[sid] = {**session, 'chat_ids': list(chat_ids)} + for cid in chat_ids: + await publish_chat_presence(cid) @sio.on('heartbeat') @@ -745,18 +810,25 @@ async def chat_events(sid, data): chat_id = data.get('chat_id') if not isinstance(chat_id, str) or not is_saved_chat_id(chat_id): return False - session = SESSION_POOL.get(sid) or user - chat_ids = set(session.get('chat_ids') or []) - if event_type == 'leave': - await sio.leave_room(sid, f'chat:{chat_id}') - chat_ids.discard(chat_id) - else: - reader = await Users.get_user_by_id(user['id']) - if not reader or not await Chats.get_accessible_chat_by_id(chat_id, reader): + async with SOCKET_EVENT_LOCKS.setdefault(('session', sid), asyncio.Lock()): + if event_type == 'leave': + await sio.leave_room(sid, f'chat:{chat_id}') + else: + reader = await Users.get_user_by_id(user['id']) + if not reader or not await Chats.get_accessible_chat_by_id(chat_id, reader): + return False + await sio.enter_room(sid, f'chat:{chat_id}') + session = SESSION_POOL.get(sid) + if not session: + await sio.leave_room(sid, f'chat:{chat_id}') return False - await sio.enter_room(sid, f'chat:{chat_id}') - chat_ids.add(chat_id) - SESSION_POOL[sid] = {**session, 'chat_ids': list(chat_ids)} + chat_ids = set(session.get('chat_ids') or []) + if event_type == 'leave': + chat_ids.discard(chat_id) + else: + chat_ids.add(chat_id) + SESSION_POOL[sid] = {**session, 'chat_ids': list(chat_ids)} + await publish_chat_presence(chat_id) return True if event_type == 'last_read_at': @@ -1128,9 +1200,14 @@ async def yjs_awareness_update(sid, data): @sio.event async def disconnect(sid, reason=None): + await encryption.disconnect(sid) LOCAL_AUTHENTICATED_SIDS.discard(sid) if sid in SESSION_POOL: + chat_ids = (SESSION_POOL.get(sid) or {}).get('chat_ids', []) del SESSION_POOL[sid] + for chat_id in chat_ids: + await sio.leave_room(sid, f'chat:{chat_id}') + await publish_chat_presence(chat_id) # Clean up USAGE_POOL entries for this session for model_id, connections in list(USAGE_POOL.items()): @@ -1279,7 +1356,7 @@ async def _make_channel_emitter(request_info): await _emit_channel_update(content, done, output if isinstance(output, list) else None) elif event_type == 'response:completion': - from open_webui.utils.middleware import handle_responses_streaming_event + from open_webui.utils.response import handle_responses_streaming_event data = event_data.get('data', {}) state['output'], _ = handle_responses_streaming_event(data, state['output']) @@ -1374,7 +1451,7 @@ async def get_event_emitter(request_info, update_db=True): if isinstance(data.get('output'), list): output = copy.deepcopy(data['output']) elif event_type == 'response:completion': - from open_webui.utils.middleware import handle_responses_streaming_event + from open_webui.utils.response import handle_responses_streaming_event output, _ = handle_responses_streaming_event(data, output or []) now = time.monotonic() diff --git a/backend/open_webui/socket/redis_room_channels.py b/backend/open_webui/socket/redis_room_channels.py index 73ac602ae8..0ad8c9aa5f 100644 --- a/backend/open_webui/socket/redis_room_channels.py +++ b/backend/open_webui/socket/redis_room_channels.py @@ -11,7 +11,7 @@ class AsyncRedisRoomChannelManager(AsyncRedisManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._local_room_channels = set() + self._subscription_queue = asyncio.Queue() # collision-free while namespaces contain no '#' (socket.io default '/'); rooms may contain '#' def _room_channel(self, namespace, room): @@ -20,12 +20,29 @@ class AsyncRedisRoomChannelManager(AsyncRedisManager): def basic_enter_room(self, sid, namespace, room, eio_sid=None): super().basic_enter_room(sid, namespace, room, eio_sid=eio_sid) if room is not None: - self._local_room_channels.add(self._room_channel(namespace, room)) + self._subscription_queue.put_nowait(asyncio.Event()) def basic_leave_room(self, sid, namespace, room): super().basic_leave_room(sid, namespace, room) if room is not None and room not in self.rooms.get(namespace, {}): - self._local_room_channels.discard(self._room_channel(namespace, room)) + self._subscription_queue.put_nowait(asyncio.Event()) + + async def _wait_for_rooms(self): + ready = asyncio.Event() + self._subscription_queue.put_nowait(ready) + await ready.wait() + + async def connect(self, eio_sid, namespace): + sid = await super().connect(eio_sid, namespace) + if sid is not None: + await self._wait_for_rooms() + return sid + + async def enter_room(self, sid, namespace, room, eio_sid=None): + local = self.is_connected(sid, namespace) + await super().enter_room(sid, namespace, room, eio_sid=eio_sid) + if local and room is not None: + await self._wait_for_rooms() async def _publish(self, data): if data.get('method') == 'emit' and isinstance(data.get('room'), str): @@ -57,23 +74,73 @@ class AsyncRedisRoomChannelManager(AsyncRedisManager): ) break + async def _sync_rooms(self, pubsub): + wanted = {self.channel.encode()} | { + self._room_channel(namespace, room) + for namespace, rooms in self.rooms.items() + for room in rooms + if room is not None + } + if added := wanted - (pubsub.channels.keys() - pubsub.pending_unsubscribe_channels): + await pubsub.subscribe(*added) + if removed := pubsub.channels.keys() - wanted - pubsub.pending_unsubscribe_channels: + await pubsub.unsubscribe(*removed) + + async def _listen_to_rooms(self, pubsub): + pubsub.ignore_subscribe_messages = False + self._subscription_queue.put_nowait(asyncio.Event()) + changed = asyncio.create_task(self._subscription_queue.get()) + message = asyncio.create_task(pubsub.get_message(timeout=None)) + waiters = [] + serial = 0 + try: + while True: + await asyncio.wait([message] if waiters else [message, changed], return_when=asyncio.FIRST_COMPLETED) + if not waiters and changed.done(): + waiters.append(changed.result()) + while not self._subscription_queue.empty(): + waiters.append(self._subscription_queue.get_nowait()) + changed = asyncio.create_task(self._subscription_queue.get()) + await self._sync_rooms(pubsub) + # PONG confirms Redis processed the subscriptions before a join returns. + serial += 1 + await pubsub.ping(str(serial).encode()) + if message.done(): + data = message.result() or {} + message = asyncio.create_task(pubsub.get_message(timeout=None)) + if data.get('type') == 'subscribe': + # redis-py can reconnect internally and restore a pending removal. + self._subscription_queue.put_nowait(asyncio.Event()) + elif data.get('type') == 'pong' and data['data'] == str(serial).encode(): + for ready in waiters: + ready.set() + waiters.clear() + else: + yield data + finally: + if changed.done() and not changed.cancelled(): + waiters.append(changed.result()) + for ready in waiters: + self._subscription_queue.put_nowait(ready) + message.cancel() + changed.cancel() + await asyncio.gather(message, changed, return_exceptions=True) + await pubsub.aclose() + async def _redis_listen_with_retries(self): _, error = self._get_redis_module_and_error() retry_sleep = 1 - subscribed = False while True: try: - if not subscribed: - self._redis_connect() - await self.pubsub.subscribe(self.channel) - await self.pubsub.psubscribe(f'{self.channel}#*') - retry_sleep = 1 - async for message in self.pubsub.listen(): + self._redis_connect() + await self.pubsub.subscribe(self.channel) + retry_sleep = 1 + async for message in self._listen_to_rooms(self.pubsub): yield message except error as exc: if isinstance(exc, NoPermissionError): self._get_logger().error( - 'Redis denied subscribing: %s. Check SUBSCRIBE/PSUBSCRIBE permissions and channel ACLs ' + 'Redis denied subscribing: %s. Check SUBSCRIBE/UNSUBSCRIBE/PING permissions and channel ACLs ' '(&%s and &%s#*). To disable room channels, set ' 'WEBSOCKET_REDIS_ROOM_CHANNELS=False on every instance and fully restart the fleet.', exc, @@ -84,18 +151,12 @@ class AsyncRedisRoomChannelManager(AsyncRedisManager): f'Cannot receive from redis... retrying in {retry_sleep} secs', extra={'redis_exception': str(exc)}, ) - subscribed = False await asyncio.sleep(retry_sleep) retry_sleep *= 2 if retry_sleep > 60: retry_sleep = 60 async def _listen(self): - main_channel = self.channel.encode() async for message in self._redis_listen_with_retries(): - if 'data' not in message: - continue - if (message['type'] == 'message' and message['channel'] == main_channel) or ( - message['type'] == 'pmessage' and message['channel'] in self._local_room_channels - ): + if message.get('type') == 'message': yield message['data'] diff --git a/backend/open_webui/tasks.py b/backend/open_webui/tasks.py index 05f622ef06..94b247badb 100644 --- a/backend/open_webui/tasks.py +++ b/backend/open_webui/tasks.py @@ -2,12 +2,16 @@ import asyncio import logging from contextlib import suppress +from itertools import groupby from uuid import uuid4 from redis.asyncio import Redis +from redis.exceptions import RedisError from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL, REDIS_TASK_TTL from open_webui.utils.json_codec import JSONCodec, dumps_bytes +from open_webui.utils.misc import get_output_text +from open_webui.utils.response import get_response_delta_key, handle_responses_streaming_event log = logging.getLogger(__name__) @@ -99,6 +103,7 @@ async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None): pipe.delete(f'{REDIS_TASKS_KEY}:{task_id}') pipe.hdel(REDIS_TASKS_KEY, task_id) pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id) + pipe.delete(f'{REDIS_RESPONSE_STREAMS_KEY}:{task_id}') if item_id: pipe.srem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) await pipe.execute() @@ -201,26 +206,62 @@ async def save_response_stream( task_id: str | None, chat_id: str | None, message_id: str | None, - content: str, output: list, -): +) -> int: + """Save a snapshot and return its byte size; zero means it needs to be retried.""" if not task_id or not chat_id or not message_id: - return + return 0 data = { 'chat_id': chat_id, 'message_id': message_id, - 'content': content, 'output': output, } if redis: - await redis.hset(REDIS_RESPONSE_STREAMS_KEY, task_id, dumps_bytes(data)) - if REDIS_RESPONSE_STREAM_TTL > 0: - with suppress(Exception): - await redis.hexpire(REDIS_RESPONSE_STREAMS_KEY, REDIS_RESPONSE_STREAM_TTL, task_id) + snapshot = dumps_bytes(data) + b'\n' + try: + await redis.set( + f'{REDIS_RESPONSE_STREAMS_KEY}:{task_id}', + snapshot, + ex=REDIS_RESPONSE_STREAM_TTL if REDIS_RESPONSE_STREAM_TTL > 0 else None, + ) + except RedisError: + return 0 + return len(snapshot) else: response_streams[task_id] = data + return 0 + + +async def append_response_stream(redis: Redis, task_id: str, delta: bytes, size: int) -> bool: + """Append only to the snapshot we last wrote, including after a Redis retry.""" + try: + return bool( + await redis.eval( + "if redis.call('strlen', KEYS[1]) ~= tonumber(ARGV[1]) then return 0 end " + "redis.call('append', KEYS[1], ARGV[2]) " + "if tonumber(ARGV[3]) > 0 then redis.call('expire', KEYS[1], ARGV[3]) end return 1", + 1, + f'{REDIS_RESPONSE_STREAMS_KEY}:{task_id}', + size, + delta, + REDIS_RESPONSE_STREAM_TTL, + ) + ) + except RedisError: + return False + + +def load_response_stream(value: str | bytes) -> dict: + lines = value.split(b'\n' if isinstance(value, bytes) else '\n') + snapshot, *deltas = (JSONCodec.loads(line) for line in lines if line) + # Join adjacent text deltas before applying them, avoiding a full text copy per token. + for _, events in groupby(deltas, key=get_response_delta_key): + first, *rest = events + delta = {**first, 'delta': first['delta'] + ''.join(event['delta'] for event in rest)} + snapshot['output'], _ = handle_responses_streaming_event(delta, snapshot['output']) + return snapshot async def get_response_streams_by_chat_id(redis, chat_id: str) -> list[dict]: @@ -229,21 +270,31 @@ async def get_response_streams_by_chat_id(redis, chat_id: str) -> list[dict]: return [] if redis: - values = await redis.hmget(REDIS_RESPONSE_STREAMS_KEY, task_ids) + pipe = redis.pipeline(transaction=False) + for task_id in task_ids: + pipe.get(f'{REDIS_RESPONSE_STREAMS_KEY}:{task_id}') + values = await pipe.execute() + # Replies started on older instances still use the shared hash during an upgrade. + if any(value is None for value in values): + legacy = await redis.hmget(REDIS_RESPONSE_STREAMS_KEY, task_ids) + values = [value if value is not None else old for value, old in zip(values, legacy)] streams = [] for value in values: if not value: continue try: - data = JSONCodec.loads(value) + data = load_response_stream(value) except Exception: continue if data.get('chat_id') == chat_id: + data['content'] = get_output_text(data['output']) streams.append(data) return streams return [ - stream for task_id in task_ids if (stream := response_streams.get(task_id)) and stream.get('chat_id') == chat_id + {**stream, 'content': get_output_text(stream['output'])} + for task_id in task_ids + if (stream := response_streams.get(task_id)) and stream.get('chat_id') == chat_id ] @@ -251,7 +302,10 @@ async def clear_response_stream(redis, task_id: str | None): if not task_id: return if redis: - await redis.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id) + pipe = redis.pipeline(transaction=False) + pipe.delete(f'{REDIS_RESPONSE_STREAMS_KEY}:{task_id}') + pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id) + await pipe.execute() else: response_streams.pop(task_id, None) diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 297b3b055c..dcbdfe5201 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -3550,7 +3550,7 @@ async def view_skill( { 'id': skill.id, 'name': snapshot['name'], - **skill_overview(snapshot['content'], file_summaries(snapshot['data']['files'])), + **await skill_overview(snapshot['content'], file_summaries(snapshot['data']['files'])), }, ensure_ascii=False, ) @@ -3573,7 +3573,7 @@ async def read_skill_file( :param id: Skill ID. :param path: Relative path within the skill. :param offset: Character offset for paging text. - :param max_chars: Maximum characters to return, up to 20000 (default 10000). + :param max_chars: Maximum characters to return, capped by the admin skill content limit (default 10000). """ try: from types import SimpleNamespace @@ -3627,7 +3627,7 @@ async def read_skill_file( return JSONCodec.dumps( { 'path': path, - **skill_content_page(file['content'], offset, max_chars), + **await skill_content_page(file['content'], offset, max_chars), }, ensure_ascii=False, ) diff --git a/backend/open_webui/utils/access_control/__init__.py b/backend/open_webui/utils/access_control/__init__.py index da690e55a5..03dad016f1 100644 --- a/backend/open_webui/utils/access_control/__init__.py +++ b/backend/open_webui/utils/access_control/__init__.py @@ -1,4 +1,5 @@ import logging +from copy import deepcopy from typing import Any from open_webui.config import DEFAULT_USER_PERMISSIONS @@ -12,7 +13,6 @@ from open_webui.models.access_grants import ( ) from open_webui.models.groups import Groups from open_webui.models.users import UserModel -from open_webui.utils.json_codec import JSONCodec from sqlalchemy.ext.asyncio import AsyncSession log = logging.getLogger(__name__) @@ -25,7 +25,7 @@ def fill_missing_permissions(permissions: dict[str, Any], default_permissions: d """ for key, value in default_permissions.items(): if key not in permissions: - permissions[key] = value + permissions[key] = deepcopy(value) elif isinstance(value, dict) and isinstance(permissions[key], dict): # Both are nested dictionaries permissions[key] = fill_missing_permissions(permissions[key], value) @@ -61,15 +61,12 @@ async def get_permissions( user_groups = await Groups.get_groups_by_member_id(user_id, db=db, include_inherited=True) # Deep copy default permissions to avoid modifying the original dict - permissions = JSONCodec.loads(JSONCodec.dumps(default_permissions)) + permissions = fill_missing_permissions(deepcopy(default_permissions or {}), DEFAULT_USER_PERMISSIONS) # Combine permissions from all user groups for group in user_groups: permissions = combine_permissions(permissions, group.permissions or {}) - # Ensure all fields from default_permissions are present and filled in - permissions = fill_missing_permissions(permissions, default_permissions) - return permissions @@ -105,7 +102,7 @@ async def has_permission( return True # Check default permissions afterward if the group permissions don't allow it - default_permissions = fill_missing_permissions(default_permissions, DEFAULT_USER_PERMISSIONS) + default_permissions = fill_missing_permissions(deepcopy(default_permissions or {}), DEFAULT_USER_PERMISSIONS) return get_permission(default_permissions, permission_hierarchy) diff --git a/backend/open_webui/utils/audit.py b/backend/open_webui/utils/audit.py index e5ff7564c1..b1d996eca6 100644 --- a/backend/open_webui/utils/audit.py +++ b/backend/open_webui/utils/audit.py @@ -264,8 +264,8 @@ class AuditLoggingMiddleware: return False # Do NOT skip logging for auth endpoints # Skip logging if the request is not authenticated - # Check both Authorization header (API keys) and token cookie (browser sessions) - if not request.headers.get('authorization') and not request.cookies.get('token'): + # Token is set by AppHTTPMiddleware from any accepted credential + if not getattr(request.state, 'token', None): return True # Whitelist mode: only log paths that match included_paths diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index ea4b4e7681..9d88eb9a92 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -81,7 +81,7 @@ from open_webui.socket.main import ( get_event_call, get_event_emitter, ) -from open_webui.tasks import clear_response_stream, save_response_stream +from open_webui.tasks import append_response_stream, clear_response_stream, save_response_stream from open_webui.utils.access_control import has_permission from open_webui.utils.access_control.files import get_owner_accessible_folder_files from open_webui.utils.access_control.folders import has_folder_access @@ -102,7 +102,7 @@ from open_webui.utils.filter import ( get_filter_functions, process_filter_functions, ) -from open_webui.utils.json_codec import JSONCodec +from open_webui.utils.json_codec import JSONCodec, dumps_bytes from open_webui.utils.memory import add_memory_context, review_memory_after_turn from open_webui.utils.misc import ( add_or_update_system_message, @@ -129,7 +129,12 @@ from open_webui.utils.misc import ( from open_webui.utils.oauth import get_system_oauth_token from open_webui.utils.payload import apply_params_to_form_data, apply_system_prompt_to_body, resolve_system_prompt from open_webui.utils.plugin import load_function_module_by_id -from open_webui.utils.response import merge_usage, normalize_usage +from open_webui.utils.response import ( + get_response_delta_key, + handle_responses_streaming_event, + merge_usage, + normalize_usage, +) from open_webui.utils.sanitize import sanitize_code from open_webui.utils.skills import ( apply_skills_create_prompt, @@ -587,27 +592,6 @@ def get_citation_source_from_tool_result( ] -def deep_merge(target, source): - """ - Merge source into target recursively (returning new structure). - - Dicts: Recursive merge. - - Strings: Concatenation. - - Others: Overwrite. - """ - if isinstance(target, dict) and isinstance(source, dict): - new_target = target.copy() - for k, v in source.items(): - if k in new_target: - new_target[k] = deep_merge(new_target[k], v) - else: - new_target[k] = v - return new_target - elif isinstance(target, str) and isinstance(source, str): - return target + source - else: - return source - - RESPONSE_COMPLETION_RESPONSE_FIELDS = ('error', 'id', 'output', 'usage') @@ -625,370 +609,6 @@ def get_response_completion_event_data(event: dict) -> dict: } -def handle_responses_streaming_event( - data: dict, - current_output: list, -) -> tuple[list, dict | None]: - """ - Handle Responses API streaming events in a pure functional way. - - Args: - data: The event data - current_output: List of output items (treated as immutable) - - Returns: - tuple[list, dict | None]: (new_output, metadata) - - new_output: The updated output list. - - metadata: Metadata to emit (e.g. usage), {} if update occurred, None if skip. - """ - # Default: no change - # Note: treating current_output as immutable, but avoiding full deepcopy for perf. - # We will shallow copy only if we need to modify the list structure or items. - - event_type = data.get('type', '') - - if event_type == 'response.output_item.added': - item = data.get('item', {}) - if item: - new_output = list(current_output) - output_index = data.get('output_index', len(new_output)) - existing_index = next( - ( - idx - for idx, existing in enumerate(new_output) - if (item.get('id') and existing.get('id') == item.get('id')) - or ( - item.get('call_id') - and existing.get('type') == item.get('type') - and existing.get('call_id') == item.get('call_id') - ) - ), - None, - ) - if existing_index is not None: - new_output[existing_index] = item - elif 0 <= output_index < len(new_output): - new_output.insert(output_index, item) - else: - new_output.append(item) - return new_output, None - return current_output, None - - elif event_type == 'response.content_part.added': - part = data.get('part', {}) - output_index = data.get('output_index', len(current_output) - 1) - - if current_output and 0 <= output_index < len(current_output): - new_output = list(current_output) - # Copy the item to mutate it - item = new_output[output_index].copy() - new_output[output_index] = item - - if 'content' not in item: - item['content'] = [] - else: - # Copy content list - item['content'] = list(item['content']) - - if item.get('type') == 'reasoning': - # Reasoning items should not have content parts - pass - else: - item['content'].append(part) - return new_output, None - return current_output, None - - elif event_type == 'response.reasoning_summary_part.added': - part = data.get('part', {}) - output_index = data.get('output_index', len(current_output) - 1) - - if current_output and 0 <= output_index < len(current_output): - new_output = list(current_output) - item = new_output[output_index].copy() - new_output[output_index] = item - - if 'summary' not in item: - item['summary'] = [] - else: - item['summary'] = list(item['summary']) - - item['summary'].append(part) - return new_output, None - return current_output, None - - elif event_type.startswith('response.') and event_type.endswith('.delta'): - # Generic Delta Handling - parts = event_type.split('.') - if len(parts) >= 3: - delta_type = parts[1] - delta = data.get('delta', '') - - output_index = data.get('output_index', max(len(current_output) - 1, 0)) - - # Chat Completions can start an item with a delta, without an added event. - if output_index >= len(current_output): - current_output = list(current_output) - while len(current_output) <= output_index: - current_output.append( - {'type': 'message', 'status': 'in_progress', 'role': 'assistant', 'content': []} - ) - current_output[output_index].update( - id=data.get('item_id'), - type='reasoning' - if delta_type.startswith('reasoning') - else ('function_call' if delta_type == 'function_call_arguments' else 'message'), - ) - - if current_output and 0 <= output_index < len(current_output): - new_output = list(current_output) - item = new_output[output_index].copy() - new_output[output_index] = item - item_type = item.get('type', '') - - # Determine target field and object based on delta_type and item_type - if delta_type == 'function_call_arguments': - key = 'arguments' - if item_type == 'function_call': - # Function call args are usually strings - item[key] = item.get(key, '') + str(delta) - else: - # Generic handling, refined by item type below - pass - - if item_type == 'message': - # Message items: "text"/"output_text" -> "text" - # "reasoning_text" -> Skipped (should use reasoning item) - if delta_type in ['text', 'output_text']: - key = 'text' - elif delta_type in ['reasoning_text', 'reasoning_summary_text']: - # Skip reasoning updates for message items - return new_output, None - else: - key = delta_type - - content_index = data.get('content_index', 0) - if 'content' not in item: - item['content'] = [] - else: - item['content'] = list(item['content']) - content_list = item['content'] - - while len(content_list) <= content_index: - content_list.append({'type': 'text', 'text': ''}) - - # Copy the part to mutate it - part = content_list[content_index].copy() - content_list[content_index] = part - - current_val = part.get(key) - if current_val is None: - # Initialize based on delta type - current_val = {} if isinstance(delta, dict) else '' - - part[key] = deep_merge(current_val, delta) - - elif item_type == 'reasoning': - # Reasoning items: "reasoning_text"/"reasoning_summary_text" -> "text" - # "text"/"output_text" -> Skipped (should use message item) - if delta_type == 'reasoning_summary_text': - # Summary updates -> item['summary'] - key = 'text' - summary_index = data.get('summary_index', 0) - if 'summary' not in item: - item['summary'] = [] - else: - item['summary'] = list(item['summary']) - summary_list = item['summary'] - - while len(summary_list) <= summary_index: - summary_list.append({'type': 'summary_text', 'text': ''}) - - part = summary_list[summary_index].copy() - summary_list[summary_index] = part - - target_val = part.get(key, '') - part[key] = deep_merge(target_val, delta) - - elif delta_type == 'reasoning_text': - # Reasoning body updates -> item['content'] - key = 'text' - content_index = data.get('content_index', 0) - if 'content' not in item: - item['content'] = [] - else: - item['content'] = list(item['content']) - content_list = item['content'] - - while len(content_list) <= content_index: - # Reasoning content parts default to text - content_list.append({'type': 'text', 'text': ''}) - - part = content_list[content_index].copy() - content_list[content_index] = part - - target_val = part.get(key, '') - part[key] = deep_merge(target_val, delta) - - elif delta_type in ['text', 'output_text']: - return new_output, None - else: - # Fallback just in case other deltas target reasoning? - pass - - else: - # Fallback for other item types - if delta_type in ['text', 'output_text']: - key = 'text' - else: - key = delta_type - - current_val = item.get(key) - if current_val is None: - current_val = {} if isinstance(delta, dict) else '' - item[key] = deep_merge(current_val, delta) - - return new_output, None - - return current_output, None - - elif event_type == 'response.output_item.done': - # Delta Event: Output item complete - item = data.get('item') - output_index = data.get('output_index', len(current_output) - 1) - - new_output = list(current_output) - if item and 0 <= output_index < len(current_output): - new_output[output_index] = item - elif item: - new_output.append(item) - return new_output, {} - - elif event_type.startswith('response.') and event_type.endswith('.done'): - # Delta Events: response.content_part.done, response.text.done, etc. - parts = event_type.split('.') - if len(parts) >= 3: - type_name = parts[1] - - # 1. Handle specific Delta "done" signals - if type_name == 'content_part': - # "Signaling that no further changes will occur to a content part" - # If payloads contains the full part, we could update it. - # Usually purely signaling in standard implementation, but we check payload. - part = data.get('part') - output_index = data.get('output_index', len(current_output) - 1) - - if part and current_output and 0 <= output_index < len(current_output): - new_output = list(current_output) - item = new_output[output_index].copy() - new_output[output_index] = item - - if 'content' in item: - item['content'] = list(item['content']) - content_index = data.get('content_index', len(item['content']) - 1) - if 0 <= content_index < len(item['content']): - item['content'][content_index] = part - return new_output, {} - return current_output, None - - elif type_name == 'reasoning_summary_part': - part = data.get('part') - output_index = data.get('output_index', len(current_output) - 1) - - if part and current_output and 0 <= output_index < len(current_output): - new_output = list(current_output) - item = new_output[output_index].copy() - new_output[output_index] = item - - if 'summary' in item: - item['summary'] = list(item['summary']) - summary_index = data.get('summary_index', len(item['summary']) - 1) - if 0 <= summary_index < len(item['summary']): - item['summary'][summary_index] = part - return new_output, {} - return current_output, None - - # 2. Generic Field Done (text.done, audio.done) - if type_name not in ['completed', 'failed']: - output_index = data.get('output_index', len(current_output) - 1) - if current_output and 0 <= output_index < len(current_output): - key = ( - 'text' - if type_name - in [ - 'text', - 'output_text', - 'reasoning_text', - 'reasoning_summary_text', - ] - else type_name - ) - if type_name == 'function_call_arguments': - key = 'arguments' - - if key in data: - final_value = data[key] - new_output = list(current_output) - item = new_output[output_index].copy() - new_output[output_index] = item - item_type = item.get('type', '') - - if type_name == 'function_call_arguments': - if item_type == 'function_call': - item['arguments'] = final_value - elif item_type == 'message': - content_index = data.get('content_index', 0) - if 'content' in item: - item['content'] = list(item['content']) - if len(item['content']) > content_index: - part = item['content'][content_index].copy() - item['content'][content_index] = part - part[key] = final_value - elif item_type == 'reasoning': - item['status'] = 'completed' - else: - item[key] = final_value - - return new_output, {} - - return current_output, None - - elif event_type == 'response.completed': - # State Machine Event: Completed - response_data = data.get('response', {}) - final_output = response_data.get('output') - - # Some providers send an empty output on response.completed despite having streamed items - new_output = final_output if final_output else current_output - - # Ensure reasoning items are marked as completed in the final output - if new_output: - for item in new_output: - if item.get('type') == 'reasoning' and item.get('status') != 'completed': - item['status'] = 'completed' - - return new_output, { - 'usage': response_data.get('usage'), - 'done': True, - 'response_id': response_data.get('id'), - } - - elif event_type == 'response.in_progress': - # State Machine Event: In Progress - # We could extract metadata if needed, but for now just acknowledge iteration - return current_output, None - - elif event_type == 'response.failed': - # State Machine Event: Failed - error = data.get('response', {}).get('error', {}) - return current_output, {'error': error} - - elif event_type == 'error': - return current_output, {'error': data} - - else: - return current_output, None - - def get_source_context(sources: list, source_ids: dict = None, include_content: bool = True) -> str: """ Build tag context string from citation sources. @@ -2439,6 +2059,7 @@ async def process_chat_payload(request, form_data, user, metadata, model): # Keep the template before apply_params_to_form_data consumes the model parameters. model_system_prompt_template = (form_data.get('params') or {}).get('system') + payload_tools = form_data.get('tools', None) # snapshot before filters form_data = apply_params_to_form_data(form_data, model) log.debug('form_data: %s', form_data) @@ -2450,7 +2071,6 @@ async def process_chat_payload(request, form_data, user, metadata, model): chat_id = metadata.get('chat_id') user_message_id = metadata.get('user_message_id') pending_assistant_message = None - payload_tools = form_data.get('tools', None) # snapshot before filters chat = await Chats.get_chat_by_id(chat_id) if is_saved_chat_id(chat_id) else None is_note_chat = bool(chat and (chat.meta or {}).get('internal') is True and (chat.meta or {}).get('type') == 'note') use_builtin_tools = is_note_chat or ( @@ -2833,18 +2453,15 @@ async def process_chat_payload(request, form_data, user, metadata, model): # Otherwise, save any tools that filter inlets added for merging later. inlet_filter_tools = None if payload_tools is not None else form_data.get('tools', None) - # Mentioned skills get bounded content; selected/default skills can be loaded through view_skill. + # Selected and mentioned skills get bounded content; other skills remain available through view_skill. chat_context = metadata.get('chat_context') or {} metadata['chat_context'] = chat_context skill_versions = chat_context.setdefault('skill_versions', {}) mentioned_skill_ids = set(chat_context.get('mentioned_skill_ids') or []) | extract_skill_ids_from_messages( form_data.get('messages', []) ) - skill_ids = sorted( - set(form_data.pop('skill_ids', None) or []) - | set(model.get('info', {}).get('meta', {}).get('skillIds', [])) - | mentioned_skill_ids - ) + activated_skill_ids = set(form_data.pop('skill_ids', None) or []) | mentioned_skill_ids + skill_ids = sorted(activated_skill_ids | set(model.get('info', {}).get('meta', {}).get('skillIds', []))) available_skills = [] terminal_skills = [] if is_note_chat: @@ -2899,7 +2516,7 @@ async def process_chat_payload(request, form_data, user, metadata, model): skill_manifest = '' for skill in available_skills: - if skill.id in mentioned_skill_ids or not use_builtin_tools or not model_builtin_tools.get('skills', True): + if skill.id in activated_skill_ids or not use_builtin_tools or not model_builtin_tools.get('skills', True): from open_webui.models.skills import get_skill_snapshot version_id = skill_versions.get(skill.id) or skill.version_id @@ -2911,7 +2528,7 @@ async def process_chat_payload(request, form_data, user, metadata, model): continue skill_versions[skill.id] = version_id skill_tools_enabled = use_builtin_tools and model_builtin_tools.get('skills', True) - overview = skill_overview( + overview = await skill_overview( snapshot['content'], [f['path'] for f in snapshot['data']['files'] if f['path'] != 'SKILL.md'], tools_enabled=skill_tools_enabled, @@ -2954,7 +2571,7 @@ async def process_chat_payload(request, form_data, user, metadata, model): for skill in terminal_skills: sid = skill['id'] - if sid in mentioned_skill_ids or not use_builtin_tools or not model_builtin_tools.get('skills', True): + if sid in activated_skill_ids or not use_builtin_tools or not model_builtin_tools.get('skills', True): skill_name = unquote(sid.removeprefix(terminal_skill_prefix)) loaded = await get_terminal_skill( request, @@ -4898,6 +4515,40 @@ async def streaming_chat_response_handler(response, ctx): ] return prior_output + output if prior_output else output + stream_size = 0 + snapshot_size = 0 + + async def save_current_response_stream(): + nonlocal stream_size, snapshot_size + stream_size = snapshot_size = await save_response_stream( + request.app.state.redis, + response_stream_task_id, + chat_id, + metadata.get('message_id'), + full_output(), + ) + + async def save_response_delta(delta): + nonlocal stream_size + if not stream_size or not request.app.state.redis: + return + # Continue merges items; code blocks write to `code`, not the event's text field. + if ( + continuing + or not isinstance(delta.get('delta'), str) + or any(item.get('type') == 'open_webui:code_interpreter' for item in output) + ): + stream_size = 0 + return + line = dumps_bytes(delta) + b'\n' + if stream_size + len(line) <= snapshot_size * 2 and await append_response_stream( + request.app.state.redis, response_stream_task_id, line, stream_size + ): + stream_size += len(line) + else: + # Snapshot after the whole provider chunk, which may update several items. + stream_size = 0 + def get_message_error_content(error): if isinstance(error, HTTPException): error = error.detail @@ -4978,6 +4629,7 @@ async def streaming_chat_response_handler(response, ctx): nonlocal output nonlocal prior_output nonlocal last_response_id + nonlocal stream_size response_tool_calls = [] response_usage = {} @@ -4991,45 +4643,6 @@ async def streaming_chat_response_handler(response, ctx): last_delta_type = None last_delta_key = None - joined_content = '' - joined_part_count = 0 - - async def save_current_response_stream(stream_output: list | None = None): - nonlocal joined_content - nonlocal joined_part_count - - if not chat_id or not metadata.get('message_id'): - return - - # content_parts is append-only, so its length tells us when the join is stale - if joined_part_count != len(content_parts): - joined_content = ''.join(content_parts) - joined_part_count = len(content_parts) - - current_stream_output = stream_output if stream_output is not None else full_output() - await save_response_stream( - request.app.state.redis, - response_stream_task_id, - chat_id, - metadata.get('message_id'), - get_output_text(current_stream_output) - if continuing - else joined_content or get_output_text(current_stream_output), - current_stream_output, - ) - - def get_response_delta_key(delta_data: dict): - event_type = delta_data.get('type', '') - if not event_type.startswith('response.') or not event_type.endswith('.delta'): - return None - return ( - event_type, - delta_data.get('item_id'), - delta_data.get('output_index'), - delta_data.get('content_index'), - delta_data.get('summary_index'), - ) - def get_response_data_with_full_output_index(response_data: dict): if prior_output and isinstance(response_data.get('output_index'), int): return { @@ -5061,7 +4674,6 @@ async def streaming_chat_response_handler(response, ctx): else last_delta_data, } ) - await save_current_response_stream() delta_count = 0 last_delta_data = None last_delta_type = None @@ -5074,6 +4686,7 @@ async def streaming_chat_response_handler(response, ctx): nonlocal last_delta_key delta_data = get_response_data_with_full_output_index(delta_data) + await save_response_delta(delta_data) delta_key = get_response_delta_key(delta_data) if ( last_delta_data @@ -5095,7 +4708,8 @@ async def streaming_chat_response_handler(response, ctx): if delta_count >= delta_chunk_size: await flush_pending_delta_data(delta_chunk_size) - async def emit_response_completion_event(response_data: dict, stream_output: list | None = None): + async def emit_response_completion_event(response_data: dict): + nonlocal stream_size if response_data.get('type', '').endswith('.delta'): await queue_pending_delta_data( response_data, @@ -5113,7 +4727,7 @@ async def streaming_chat_response_handler(response, ctx): else get_response_completion_event_data(response_data), } ) - await save_current_response_stream(stream_output) + stream_size = 0 filter_extra_params = {'__body__': form_data, **extra_params} if filter_functions else None @@ -5433,6 +5047,8 @@ async def streaming_chat_response_handler(response, ctx): func = tc.get('function', {}) if call_id in output_by_call_id: output_index, item = output_by_call_id[call_id] + if func.get('name', item.get('name', '')) != item.get('name'): + stream_size = 0 item['name'] = func.get('name', item.get('name', '')) item['arguments'] = func.get('arguments', item.get('arguments', '')) item['status'] = 'in_progress' @@ -5484,7 +5100,6 @@ async def streaming_chat_response_handler(response, ctx): } ) - await save_current_response_stream() data = None delta_type = 'tool_call' @@ -5625,7 +5240,7 @@ async def streaming_chat_response_handler(response, ctx): await event_emitter( {'type': 'chat:completion', 'data': {'output': full_output()}} ) - await save_current_response_stream() + stream_size = 0 data = None if value: @@ -5813,7 +5428,7 @@ async def streaming_chat_response_handler(response, ctx): await event_emitter( {'type': 'chat:completion', 'data': {'output': full_output()}} ) - await save_current_response_stream() + stream_size = 0 data = None if delta and data: @@ -5833,7 +5448,11 @@ async def streaming_chat_response_handler(response, ctx): pass else: log.debug('Error: %s', e) + stream_size = 0 continue + finally: + if not stream_size or not request.app.state.redis: + await save_current_response_stream() await flush_pending_delta_data() if output: @@ -5923,6 +5542,7 @@ async def streaming_chat_response_handler(response, ctx): if responses_api_tool_calls: tool_calls.append(_split_tool_calls(responses_api_tool_calls)) + await save_current_response_stream() return normalize_usage(response_usage) output_start = len(prior_output) @@ -5955,6 +5575,7 @@ async def streaming_chat_response_handler(response, ctx): base_system_prompt = get_content_from_message(system_message) if system_message else None async def emit_output(): + await save_current_response_stream() # Channels publish whole messages; Continue can merge into the preceding item. snapshot = continuing or (metadata.get('chat_id') or '').startswith('channel:') frontend_output = [] @@ -6337,9 +5958,26 @@ async def streaming_chat_response_handler(response, ctx): ) try: - messages, context_summary, compacted = await compact_messages_for_request( - request, user, messages, metadata, model_id, request.app.state.MODELS - ) + user_messages = [ + (message, message.get('content')) + for message in messages + if user_prompt + and message.get('role') == 'user' + and message.get('content') == user_prompt.get('content') + ] + try: + # Summarize the user's text, then restore source context for the model. + for message, _ in user_messages: + message['content'] = copy.deepcopy(message.get('content')) + set_last_user_message_content( + metadata.get('base_user_prompt') or user_message, [message] + ) + messages, context_summary, compacted = await compact_messages_for_request( + request, user, messages, metadata, model_id, request.app.state.MODELS + ) + finally: + for message, content in user_messages: + message['content'] = content if compacted: if user_prompt is not None: messages = [message for message in messages if message != user_prompt] diff --git a/backend/open_webui/utils/oauth.py b/backend/open_webui/utils/oauth.py index 76f1ebb346..ee690088b6 100644 --- a/backend/open_webui/utils/oauth.py +++ b/backend/open_webui/utils/oauth.py @@ -1186,8 +1186,14 @@ class OAuthClientManager: if should_send_oauth_resource(client_info): refresh_data['resource'] = client_info.resource + client_auth = None if hasattr(client, 'client_secret') and client.client_secret: - refresh_data['client_secret'] = client.client_secret + if client.client_kwargs.get('token_endpoint_auth_method') == 'client_secret_basic': + client_auth = aiohttp.BasicAuth(client.client_id, client.client_secret) + # Some providers reject a body client_id next to Basic as a second credential (#19415) + del refresh_data['client_id'] + else: + refresh_data['client_secret'] = client.client_secret # Add scope if available in client kwargs (some providers require it on refresh) if ( @@ -1202,6 +1208,7 @@ class OAuthClientManager: async with session_http.post( token_endpoint, data=refresh_data, + auth=client_auth, headers={'Content-Type': 'application/x-www-form-urlencoded'}, ssl=AIOHTTP_CLIENT_SESSION_SSL, ) as r: @@ -1538,9 +1545,15 @@ class OAuthManager: 'refresh_token': token_data['refresh_token'], 'client_id': client.client_id, } + client_auth = None # Add client_secret if available (some providers require it) if hasattr(client, 'client_secret') and client.client_secret: - refresh_data['client_secret'] = client.client_secret + if client.client_kwargs.get('token_endpoint_auth_method') == 'client_secret_basic': + client_auth = aiohttp.BasicAuth(client.client_id, client.client_secret) + if not OAUTH_ACCESS_TOKEN_REQUEST_INCLUDE_CLIENT_ID: + del refresh_data['client_id'] + else: + refresh_data['client_secret'] = client.client_secret # Add scope if available in client kwargs (some providers require it on refresh) if ( @@ -1555,6 +1568,7 @@ class OAuthManager: async with session_http.post( token_endpoint, data=refresh_data, + auth=client_auth, headers={'Content-Type': 'application/x-www-form-urlencoded'}, ssl=AIOHTTP_CLIENT_SESSION_SSL, ) as r: diff --git a/backend/open_webui/utils/response.py b/backend/open_webui/utils/response.py index ab88d56d3f..6ca4957448 100644 --- a/backend/open_webui/utils/response.py +++ b/backend/open_webui/utils/response.py @@ -342,3 +342,401 @@ def convert_embedding_response_ollama_to_openai(response) -> dict: # Fallback: return as is if unrecognized return response + + +def get_response_delta_key(delta_data: dict): + event_type = delta_data.get('type', '') + if not event_type.startswith('response.') or not event_type.endswith('.delta'): + return None + return ( + event_type, + delta_data.get('item_id'), + delta_data.get('output_index'), + delta_data.get('content_index'), + delta_data.get('summary_index'), + ) + + +def deep_merge(target, source): + """ + Merge source into target recursively (returning new structure). + - Dicts: Recursive merge. + - Strings: Concatenation. + - Others: Overwrite. + """ + if isinstance(target, dict) and isinstance(source, dict): + new_target = target.copy() + for k, v in source.items(): + if k in new_target: + new_target[k] = deep_merge(new_target[k], v) + else: + new_target[k] = v + return new_target + elif isinstance(target, str) and isinstance(source, str): + return target + source + else: + return source + + +def handle_responses_streaming_event( + data: dict, + current_output: list, +) -> tuple[list, dict | None]: + """ + Handle Responses API streaming events in a pure functional way. + + Args: + data: The event data + current_output: List of output items (treated as immutable) + + Returns: + tuple[list, dict | None]: (new_output, metadata) + - new_output: The updated output list. + - metadata: Metadata to emit (e.g. usage), {} if update occurred, None if skip. + """ + # Default: no change + # Note: treating current_output as immutable, but avoiding full deepcopy for perf. + # We will shallow copy only if we need to modify the list structure or items. + + event_type = data.get('type', '') + + if event_type == 'response.output_item.added': + item = data.get('item', {}) + if item: + new_output = list(current_output) + output_index = data.get('output_index', len(new_output)) + existing_index = next( + ( + idx + for idx, existing in enumerate(new_output) + if (item.get('id') and existing.get('id') == item.get('id')) + or ( + item.get('call_id') + and existing.get('type') == item.get('type') + and existing.get('call_id') == item.get('call_id') + ) + ), + None, + ) + if existing_index is not None: + new_output[existing_index] = item + elif 0 <= output_index < len(new_output): + new_output.insert(output_index, item) + else: + new_output.append(item) + return new_output, None + return current_output, None + + elif event_type == 'response.content_part.added': + part = data.get('part', {}) + output_index = data.get('output_index', len(current_output) - 1) + + if current_output and 0 <= output_index < len(current_output): + new_output = list(current_output) + # Copy the item to mutate it + item = new_output[output_index].copy() + new_output[output_index] = item + + if 'content' not in item: + item['content'] = [] + else: + # Copy content list + item['content'] = list(item['content']) + + if item.get('type') == 'reasoning': + # Reasoning items should not have content parts + pass + else: + item['content'].append(part) + return new_output, None + return current_output, None + + elif event_type == 'response.reasoning_summary_part.added': + part = data.get('part', {}) + output_index = data.get('output_index', len(current_output) - 1) + + if current_output and 0 <= output_index < len(current_output): + new_output = list(current_output) + item = new_output[output_index].copy() + new_output[output_index] = item + + if 'summary' not in item: + item['summary'] = [] + else: + item['summary'] = list(item['summary']) + + item['summary'].append(part) + return new_output, None + return current_output, None + + elif event_type.startswith('response.') and event_type.endswith('.delta'): + # Generic Delta Handling + parts = event_type.split('.') + if len(parts) >= 3: + delta_type = parts[1] + delta = data.get('delta', '') + + output_index = data.get('output_index', max(len(current_output) - 1, 0)) + + # Chat Completions can start an item with a delta, without an added event. + if output_index >= len(current_output): + current_output = list(current_output) + while len(current_output) <= output_index: + current_output.append( + {'type': 'message', 'status': 'in_progress', 'role': 'assistant', 'content': []} + ) + current_output[output_index].update( + id=data.get('item_id'), + type='reasoning' + if delta_type.startswith('reasoning') + else ('function_call' if delta_type == 'function_call_arguments' else 'message'), + ) + + if current_output and 0 <= output_index < len(current_output): + new_output = list(current_output) + item = new_output[output_index].copy() + new_output[output_index] = item + item_type = item.get('type', '') + + # Determine target field and object based on delta_type and item_type + if delta_type == 'function_call_arguments': + key = 'arguments' + if item_type == 'function_call': + # Function call args are usually strings + item[key] = item.get(key, '') + str(delta) + else: + # Generic handling, refined by item type below + pass + + if item_type == 'message': + # Message items: "text"/"output_text" -> "text" + # "reasoning_text" -> Skipped (should use reasoning item) + if delta_type in ['text', 'output_text']: + key = 'text' + elif delta_type in ['reasoning_text', 'reasoning_summary_text']: + # Skip reasoning updates for message items + return new_output, None + else: + key = delta_type + + content_index = data.get('content_index', 0) + if 'content' not in item: + item['content'] = [] + else: + item['content'] = list(item['content']) + content_list = item['content'] + + while len(content_list) <= content_index: + content_list.append({'type': 'text', 'text': ''}) + + # Copy the part to mutate it + part = content_list[content_index].copy() + content_list[content_index] = part + + current_val = part.get(key) + if current_val is None: + # Initialize based on delta type + current_val = {} if isinstance(delta, dict) else '' + + part[key] = deep_merge(current_val, delta) + + elif item_type == 'reasoning': + # Reasoning items: "reasoning_text"/"reasoning_summary_text" -> "text" + # "text"/"output_text" -> Skipped (should use message item) + if delta_type == 'reasoning_summary_text': + # Summary updates -> item['summary'] + key = 'text' + summary_index = data.get('summary_index', 0) + if 'summary' not in item: + item['summary'] = [] + else: + item['summary'] = list(item['summary']) + summary_list = item['summary'] + + while len(summary_list) <= summary_index: + summary_list.append({'type': 'summary_text', 'text': ''}) + + part = summary_list[summary_index].copy() + summary_list[summary_index] = part + + target_val = part.get(key, '') + part[key] = deep_merge(target_val, delta) + + elif delta_type == 'reasoning_text': + # Reasoning body updates -> item['content'] + key = 'text' + content_index = data.get('content_index', 0) + if 'content' not in item: + item['content'] = [] + else: + item['content'] = list(item['content']) + content_list = item['content'] + + while len(content_list) <= content_index: + # Reasoning content parts default to text + content_list.append({'type': 'text', 'text': ''}) + + part = content_list[content_index].copy() + content_list[content_index] = part + + target_val = part.get(key, '') + part[key] = deep_merge(target_val, delta) + + elif delta_type in ['text', 'output_text']: + return new_output, None + else: + # Fallback just in case other deltas target reasoning? + pass + + else: + # Fallback for other item types + if delta_type in ['text', 'output_text']: + key = 'text' + else: + key = delta_type + + current_val = item.get(key) + if current_val is None: + current_val = {} if isinstance(delta, dict) else '' + item[key] = deep_merge(current_val, delta) + + return new_output, None + + return current_output, None + + elif event_type == 'response.output_item.done': + # Delta Event: Output item complete + item = data.get('item') + output_index = data.get('output_index', len(current_output) - 1) + + new_output = list(current_output) + if item and 0 <= output_index < len(current_output): + new_output[output_index] = item + elif item: + new_output.append(item) + return new_output, {} + + elif event_type.startswith('response.') and event_type.endswith('.done'): + # Delta Events: response.content_part.done, response.text.done, etc. + parts = event_type.split('.') + if len(parts) >= 3: + type_name = parts[1] + + # 1. Handle specific Delta "done" signals + if type_name == 'content_part': + # "Signaling that no further changes will occur to a content part" + # If payloads contains the full part, we could update it. + # Usually purely signaling in standard implementation, but we check payload. + part = data.get('part') + output_index = data.get('output_index', len(current_output) - 1) + + if part and current_output and 0 <= output_index < len(current_output): + new_output = list(current_output) + item = new_output[output_index].copy() + new_output[output_index] = item + + if 'content' in item: + item['content'] = list(item['content']) + content_index = data.get('content_index', len(item['content']) - 1) + if 0 <= content_index < len(item['content']): + item['content'][content_index] = part + return new_output, {} + return current_output, None + + elif type_name == 'reasoning_summary_part': + part = data.get('part') + output_index = data.get('output_index', len(current_output) - 1) + + if part and current_output and 0 <= output_index < len(current_output): + new_output = list(current_output) + item = new_output[output_index].copy() + new_output[output_index] = item + + if 'summary' in item: + item['summary'] = list(item['summary']) + summary_index = data.get('summary_index', len(item['summary']) - 1) + if 0 <= summary_index < len(item['summary']): + item['summary'][summary_index] = part + return new_output, {} + return current_output, None + + # 2. Generic Field Done (text.done, audio.done) + if type_name not in ['completed', 'failed']: + output_index = data.get('output_index', len(current_output) - 1) + if current_output and 0 <= output_index < len(current_output): + key = ( + 'text' + if type_name + in [ + 'text', + 'output_text', + 'reasoning_text', + 'reasoning_summary_text', + ] + else type_name + ) + if type_name == 'function_call_arguments': + key = 'arguments' + + if key in data: + final_value = data[key] + new_output = list(current_output) + item = new_output[output_index].copy() + new_output[output_index] = item + item_type = item.get('type', '') + + if type_name == 'function_call_arguments': + if item_type == 'function_call': + item['arguments'] = final_value + elif item_type == 'message': + content_index = data.get('content_index', 0) + if 'content' in item: + item['content'] = list(item['content']) + if len(item['content']) > content_index: + part = item['content'][content_index].copy() + item['content'][content_index] = part + part[key] = final_value + elif item_type == 'reasoning': + item['status'] = 'completed' + else: + item[key] = final_value + + return new_output, {} + + return current_output, None + + elif event_type == 'response.completed': + # State Machine Event: Completed + response_data = data.get('response', {}) + final_output = response_data.get('output') + + # Some providers send an empty output on response.completed despite having streamed items + new_output = final_output if final_output else current_output + + # Ensure reasoning items are marked as completed in the final output + if new_output: + for item in new_output: + if item.get('type') == 'reasoning' and item.get('status') != 'completed': + item['status'] = 'completed' + + return new_output, { + 'usage': response_data.get('usage'), + 'done': True, + 'response_id': response_data.get('id'), + } + + elif event_type == 'response.in_progress': + # State Machine Event: In Progress + # We could extract metadata if needed, but for now just acknowledge iteration + return current_output, None + + elif event_type == 'response.failed': + # State Machine Event: Failed + error = data.get('response', {}).get('error', {}) + return current_output, {'error': error} + + elif event_type == 'error': + return current_output, {'error': data} + + else: + return current_output, None diff --git a/backend/open_webui/utils/skill_files.py b/backend/open_webui/utils/skill_files.py index 220f94fa12..47a53013df 100644 --- a/backend/open_webui/utils/skill_files.py +++ b/backend/open_webui/utils/skill_files.py @@ -31,9 +31,15 @@ SKILL_GREP_EXCERPT_MAX_CHARS = 1_200 SKILL_GREP_LOCATIONS_MAX_CHARS = 2_000 -def skill_content_page(content: str, offset: int = 0, max_chars: int = SKILL_CONTENT_MAX_CHARS) -> dict: +async def skill_content_page( + content: str, offset: int = 0, max_chars: int = SKILL_CONTENT_MAX_CHARS, *, limit: int | None = None +) -> dict: + if limit is None: + from open_webui.models.config import Config + + limit = await Config.get('skills.content_max_chars') or SKILL_CONTENT_MAX_CHARS offset = max(0, offset) - end = offset + min(SKILL_CONTENT_MAX_CHARS, max(1, max_chars)) + end = offset + min(limit, max(1, max_chars)) return {'content': content[offset:end], 'next_offset': end if end < len(content) else None} @@ -196,16 +202,21 @@ def grep_skill_content(files: list[dict], pattern: str, case_insensitive: bool = return serialized -def skill_overview(content: str, files: list, field: str = 'files', tools_enabled: bool = True) -> dict: +async def skill_overview(content: str, files: list, field: str = 'files', tools_enabled: bool = True) -> dict: if not tools_enabled: return { 'content': content[:SKILL_INJECTION_MAX_CHARS], 'next_offset': SKILL_INJECTION_MAX_CHARS if len(content) > SKILL_INJECTION_MAX_CHARS else None, **bounded_skill_manifest(files, field), } - large = len(content) > SKILL_CONTENT_MAX_CHARS + from open_webui.models.config import Config + + limits = await Config.get_many('skills.content_max_chars', 'skills.preview_max_chars') + content_limit = limits.get('skills.content_max_chars') or SKILL_CONTENT_MAX_CHARS + preview_limit = limits.get('skills.preview_max_chars') or SKILL_PREVIEW_MAX_CHARS + large = len(content) > content_limit return { - **skill_content_page(content, max_chars=SKILL_PREVIEW_MAX_CHARS if large else SKILL_CONTENT_MAX_CHARS), + **await skill_content_page(content, max_chars=preview_limit if large else content_limit, limit=content_limit), **bounded_skill_manifest(files, field, SKILL_OVERVIEW_LIST_MAX_CHARS), } diff --git a/backend/open_webui/utils/terminals.py b/backend/open_webui/utils/terminals.py index b1432c50dd..1bb2369629 100644 --- a/backend/open_webui/utils/terminals.py +++ b/backend/open_webui/utils/terminals.py @@ -287,10 +287,10 @@ async def get_terminal_skill( 'description': skill.get('description'), 'directory': directory, **( - skill_overview(skill['content'], resources, 'resources', tools_enabled) + await skill_overview(skill['content'], resources, 'resources', tools_enabled) if overview else { - **skill_content_page(skill['content'], offset, max_chars), + **await skill_content_page(skill['content'], offset, max_chars), **bounded_skill_manifest(resources, 'resources'), } ), diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 19154abf6d..212d283129 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -1325,7 +1325,13 @@ async def get_tool_servers(request: Request): data = await request.app.state.redis.get(f'{REDIS_KEY_PREFIX}:tool_servers') if data is not None: tool_servers = JSONCodec.loads(data) - request.app.state.TOOL_SERVERS = tool_servers + if tool_servers or not any( + connection.get('config', {}).get('enable') and connection.get('type', 'openapi') == 'openapi' + for connection in await Config.get('tool_server.connections', []) or [] + ): + request.app.state.TOOL_SERVERS = tool_servers + else: + tool_servers = None except Exception as e: log.error(f'Error fetching tool_servers from Redis: {e}') diff --git a/crypto/encryption/.gitignore b/crypto/encryption/.gitignore new file mode 100644 index 0000000000..b83d22266a --- /dev/null +++ b/crypto/encryption/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/crypto/encryption/Cargo.lock b/crypto/encryption/Cargo.lock new file mode 100644 index 0000000000..7783d53fed --- /dev/null +++ b/crypto/encryption/Cargo.lock @@ -0,0 +1,2026 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20 0.9.1", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", + "zeroize", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-models" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee3344d349528f449816cfa85e558c439bdca5a6d8a738cfb21e69f0b2b1952f" +dependencies = [ + "hax-lib", + "pastey", + "rand", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crabgrind" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7459e07732f6de74001fcf73a3fdfbb0f368e4fd8e2392f4a2206a065e6e95" +dependencies = [ + "bindgen", + "cc", + "pkg-config", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "curve25519-dalek-derive", + "fiat-crypto 0.3.0", + "rustc_version", + "subtle", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der 0.7.10", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature 2.2.0", + "spki 0.7.3", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "signature 2.2.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek 4.1.3", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf 0.12.4", + "pem-rfc7468", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "hax-lib" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a33cb227f97ee5c419064c31d7c55a7d895f28d8019ccdadc311cc036500c4" +dependencies = [ + "hax-lib-macros", + "num-bigint", + "num-traits", +] + +[[package]] +name = "hax-lib-macros" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28da835a5f153e9f3d0cc1f42ee605a1cfa9093c93348193793d60a1b0908b51" +dependencies = [ + "hax-lib-macros-types", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "hax-lib-macros-types" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed327e9a28d6ee018a4a9ba842d4ee27e378c8a591d2beb6f42f32b24a02fcb" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "hpke-rs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812de62ab573b876c6c40d7553bae9402ae3bad95b64af06f964388eecb9b7b1" +dependencies = [ + "hpke-rs-crypto", + "hpke-rs-libcrux", + "hpke-rs-rust-crypto", + "libcrux-sha3", + "log", + "serde", + "subtle", + "tls_codec", + "zeroize", +] + +[[package]] +name = "hpke-rs-crypto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c461da2e0c9c93d875b597f0c78214fb0d2d5c57834b2ef2a2fa057fa20e24" +dependencies = [ + "rand", + "zeroize", +] + +[[package]] +name = "hpke-rs-libcrux" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1ace95ef11fbbd84527eada5fc1304f66720fc4c008db30d888d62d3c52613" +dependencies = [ + "hpke-rs-crypto", + "libcrux-aead", + "libcrux-ecdh", + "libcrux-hkdf", + "libcrux-kem", + "libcrux-traits", + "rand", + "rand_chacha 0.10.0", + "rand_core 0.10.1", + "zeroize", +] + +[[package]] +name = "hpke-rs-rust-crypto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a606ac19851da841862ef5f39d26d6227bfcfb4047417801a66c1f6e6652d71" +dependencies = [ + "aes-gcm", + "chacha20poly1305", + "hkdf 0.13.0", + "hpke-rs-crypto", + "k256", + "ml-kem", + "p256", + "p384", + "rand", + "rand_chacha 0.10.0", + "rand_core 0.10.1", + "sha2 0.11.1", + "subtle", + "x-wing", + "x25519-dalek 2.0.1", + "zeroize", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "ctutils", + "typenum", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "elliptic-curve", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + +[[package]] +name = "kem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01737161ba802849cfd486b5bd209d38ba4943494c249a8126005170c7621edd" +dependencies = [ + "crypto-common 0.2.2", + "rand_core 0.10.1", +] + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libcrux-aead" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22acbe68be84b7b41aaba6e39b87036fc4324dee628d94773750bf3d7e906d69" +dependencies = [ + "libcrux-aes", + "libcrux-chacha20poly1305", + "libcrux-secrets", + "libcrux-traits", +] + +[[package]] +name = "libcrux-aes" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57cc95b11dbc797b1e169467b1fc4205c4e6ee2ce516a035ad7342ce5f6ae971" +dependencies = [ + "libcrux-intrinsics", + "libcrux-platform", + "libcrux-secrets", + "libcrux-traits", +] + +[[package]] +name = "libcrux-chacha20poly1305" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537e6eee5cdc9a980014d058784b0be09614f0596df789b114f7833aa9f35d75" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-macros", + "libcrux-poly1305", + "libcrux-secrets", + "libcrux-traits", +] + +[[package]] +name = "libcrux-curve25519" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4f3cfd5e31cb9745e290ee061222c380ec42884654b09fc7d12a5b0ed63e028" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-macros", + "libcrux-secrets", + "libcrux-traits", +] + +[[package]] +name = "libcrux-ecdh" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a227590b89ff55ce7cd97b5a7468c82d231db8f3b890bb4d4fb6d271e7524d7" +dependencies = [ + "libcrux-curve25519", + "libcrux-p256", + "rand", + "tls_codec", +] + +[[package]] +name = "libcrux-hacl-rs" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66106db376ae249af86911aee048cf73ea1f394d6653026fc988dd22cf2a4ace" +dependencies = [ + "libcrux-macros", +] + +[[package]] +name = "libcrux-hkdf" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc94e651dca4d47edbcdd88bb2428ce16a2bd86b7a4e7170da87b505478f9839" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-hmac", + "libcrux-secrets", +] + +[[package]] +name = "libcrux-hmac" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500ad9b32c715161b594172ca1fb11503a1c033b393ff4c0c33505ce51c1943c" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-macros", + "libcrux-sha2", + "libcrux-traits", +] + +[[package]] +name = "libcrux-intrinsics" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98a0c574d4eb81d0814bc2b91e4a433d7e0b35851b1d62eb5dd95c064f1f76d0" +dependencies = [ + "core-models", + "hax-lib", +] + +[[package]] +name = "libcrux-kem" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "541a7377fb35060892e0620982e224e47419f10da8c212453bf642dafe529691" +dependencies = [ + "libcrux-curve25519", + "libcrux-ecdh", + "libcrux-ml-kem", + "libcrux-p256", + "libcrux-sha3", + "libcrux-traits", + "rand", +] + +[[package]] +name = "libcrux-macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd6aa2dcd5be681662001b81d493f1569c6d49a32361f470b0c955465cd0338" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "libcrux-ml-kem" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8160f7d64fd2716b4fd05cc886a042f8dcda18d9206c0d506e2c67bdf97daa" +dependencies = [ + "hax-lib", + "libcrux-intrinsics", + "libcrux-platform", + "libcrux-secrets", + "libcrux-sha3", + "libcrux-traits", + "rand", + "tls_codec", +] + +[[package]] +name = "libcrux-p256" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3400732702d578be622257b98cec85e9b0cd34a67f1f06d3ebe9ae34963cdf02" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-macros", + "libcrux-secrets", + "libcrux-sha2", + "libcrux-traits", +] + +[[package]] +name = "libcrux-platform" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9e21d7ed31a92ac539bd69a8c970b183ee883872d2d19ce27036e24cb8ecc4" +dependencies = [ + "libc", +] + +[[package]] +name = "libcrux-poly1305" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a9144949845813a0b8787d08cbba47baabe59c83f3043e558e6e92385c40cc" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-macros", +] + +[[package]] +name = "libcrux-secrets" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79054fc9037cb70d6a546cf094cea7d7df06af5e49d230ba24103d27ccc886f1" +dependencies = [ + "crabgrind", + "hax-lib", +] + +[[package]] +name = "libcrux-sha2" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e46e1be0b77098cc6ce82137864936ecad3a1b9fd4303dd51202ca140dd1e" +dependencies = [ + "libcrux-hacl-rs", + "libcrux-macros", + "libcrux-traits", +] + +[[package]] +name = "libcrux-sha3" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c09f5c39afae0528e1f70a3c1e3c6ee649ec1f19dda26fc9b6ea91108fb879ef" +dependencies = [ + "hax-lib", + "libcrux-intrinsics", + "libcrux-platform", + "libcrux-traits", +] + +[[package]] +name = "libcrux-traits" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa7a21e8c2e8baa8b40f0b176740f2ca4baadd79d1b00e48d6b1e363c42085a" +dependencies = [ + "libcrux-secrets", + "rand", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "ml-dsa" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add6b9d92e496f16f4526d68ff29da1483aba4b119baeab8bed3b9e3544a6f3d" +dependencies = [ + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", + "hybrid-array", + "module-lattice", + "pkcs8 0.11.0", + "shake", + "signature 3.0.0", +] + +[[package]] +name = "ml-kem" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" +dependencies = [ + "hybrid-array", + "kem", + "module-lattice", + "rand_core 0.10.1", + "sha3 0.11.0", +] + +[[package]] +name = "module-lattice" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +dependencies = [ + "ctutils", + "hybrid-array", + "num-traits", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openmls" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ea97f8931047590bda498a9502e8fe6c9bb2bb04547a9a463618783481e0ec" +dependencies = [ + "getrandom 0.4.3", + "log", + "openmls_serialization_helpers", + "openmls_traits", + "rayon", + "serde", + "serde_bytes", + "thiserror", + "tls_codec", + "web-time", + "zeroize", +] + +[[package]] +name = "openmls_basic_credential" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd3f0c3422e7c7a8496f042b547b0c28d0793f8ba97feb401966e58196a1e40" +dependencies = [ + "ed25519-dalek", + "ml-dsa", + "openmls_traits", + "p256", + "p384", + "rand_core 0.6.4", + "serde", + "tls_codec", + "zeroize", +] + +[[package]] +name = "openmls_memory_storage" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce927945a16eaa19302ed5ccef052f01ba68f2ba7ae67e4ac4ff2fc661445364" +dependencies = [ + "log", + "openmls_traits", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "openmls_rust_crypto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36bf3fe824ead6e81d22f49a9cd7feffca3494979fac14c1248e19516e3625f1" +dependencies = [ + "aes-gcm", + "chacha20poly1305", + "ed25519-dalek", + "hkdf 0.13.0", + "hmac 0.13.0", + "hpke-rs", + "hpke-rs-crypto", + "hpke-rs-rust-crypto", + "ml-dsa", + "openmls_memory_storage", + "openmls_traits", + "p256", + "p384", + "rand_chacha 0.3.1", + "rand_core 0.10.1", + "rand_core 0.6.4", + "sha2 0.11.1", + "thiserror", + "tls_codec", +] + +[[package]] +name = "openmls_serialization_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e2cf2193541e83ce1a89aaa44c2fd4783abd54853823075a32b60ef032284c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "thiserror", +] + +[[package]] +name = "openmls_traits" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac13edad8698eb1917cbc1dab99591fa1594d84320904fe3fc5121854f95831c" +dependencies = [ + "serde", + "tls_codec", +] + +[[package]] +name = "encryption" +version = "0.1.0" +dependencies = [ + "base64", + "getrandom 0.2.17", + "openmls", + "openmls_basic_credential", + "openmls_rust_crypto", + "openmls_traits", + "serde", + "serde_json", + "sha2 0.10.9", + "tls_codec", + "wasm-bindgen", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.1", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20 0.10.2", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der 0.7.10", + "generic-array", + "pkcs8 0.10.2", + "subtle", + "zeroize", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak", +] + +[[package]] +name = "sha3" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" +dependencies = [ + "digest 0.11.3", + "keccak", + "sponge-cursor", +] + +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak", + "sponge-cursor", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.2", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tls_codec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18cc98286004cea38f717e2b03d990fc774fbfd38a82720de40e5c94365067c8" +dependencies = [ + "serde", + "serde_bytes", + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "674f41dd95f76cdeb005c83f9444e20cf43daebef37cde9e49a9ca6f4e87b423" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "x-wing" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01576e56599d01d75c5efcd5b30fa5de29e776f5a225e43fbf8c92c7302d4c8b" +dependencies = [ + "kem", + "ml-kem", + "sha3 0.12.0", + "shake", + "x25519-dalek 3.0.0", +] + +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek 4.1.3", + "rand_core 0.6.4", + "serde", + "zeroize", +] + +[[package]] +name = "x25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" +dependencies = [ + "curve25519-dalek 5.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crypto/encryption/Cargo.toml b/crypto/encryption/Cargo.toml new file mode 100644 index 0000000000..13254268e6 --- /dev/null +++ b/crypto/encryption/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "encryption" +version = "0.1.0" +edition = "2021" +publish = false +license = "MIT" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +openmls = { version = "=0.9.1", features = ["js"] } +openmls_traits = "=0.6.0" +openmls_rust_crypto = "=0.6.0" +openmls_basic_credential = "=0.6.0" +getrandom = { version = "0.2", features = ["js"] } +wasm-bindgen = "=0.2.126" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tls_codec = "0.5" +base64 = "0.22" +sha2 = "0.10" + +[profile.release] +opt-level = "s" +lto = true +codegen-units = 1 +panic = "abort" diff --git a/crypto/encryption/rust-toolchain.toml b/crypto/encryption/rust-toolchain.toml new file mode 100644 index 0000000000..c5432d939a --- /dev/null +++ b/crypto/encryption/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.95.0" +targets = ["wasm32-unknown-unknown"] +profile = "minimal" diff --git a/crypto/encryption/src/lib.rs b/crypto/encryption/src/lib.rs new file mode 100644 index 0000000000..f1c7b90028 --- /dev/null +++ b/crypto/encryption/src/lib.rs @@ -0,0 +1,267 @@ +//! The browser worker is the only owner. No persistence, export, or logging API. +use base64::{engine::general_purpose::STANDARD as B64, Engine}; +use openmls::prelude::*; +use openmls_basic_credential::SignatureKeyPair; +use openmls_rust_crypto::OpenMlsRustCrypto; +use openmls_traits::OpenMlsProvider; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use tls_codec::{Deserialize as TlsDeserialize, Serialize as TlsSerialize}; +use wasm_bindgen::prelude::*; + +const SUITE: Ciphersuite = Ciphersuite::MLS_128_DHKEMX25519_AES128GCM_SHA256_Ed25519; +const MAX_PACKET: usize = 1024 * 1024; +fn err(_: T) -> String { "Invalid session operation".into() } +fn require(ok: bool) -> Result<(), String> { if ok { Ok(()) } else { Err(err(())) } } +fn encode(v: &T) -> Result { Ok(B64.encode(v.tls_serialize_detached().map_err(err)?)) } +fn decode(v: &str) -> Result, String> { + require(v.len() <= MAX_PACKET * 4 / 3 + 4)?; + let bytes = B64.decode(v).map_err(err)?; + require(bytes.len() <= MAX_PACKET)?; + Ok(bytes) +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Identity { channel: String, session: String, user: String, participation: String } +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct Binding { identity: Identity, key: String } +fn identity(cred: &Credential) -> Result { + let basic = BasicCredential::try_from(cred.clone()).map_err(err)?; + serde_json::from_slice(basic.identity()).map_err(err) +} +fn check(cred: &Credential, key: &[u8], context: &Identity, roster: &[Binding]) -> Result { + let id = identity(cred)?; + require(id.channel == context.channel && id.session == context.session)?; + require(roster.iter().any(|b| b.identity == id && b.key == B64.encode(key)))?; + Ok(id) +} +fn roster(value: &str, context: &Identity) -> Result, String> { + let roster: Vec = serde_json::from_str(value).map_err(err)?; + require(!roster.is_empty() && roster.len() <= 128)?; + let mut users = std::collections::HashSet::new(); + let mut participations = std::collections::HashSet::new(); + for b in &roster { + require(b.identity.channel == context.channel && b.identity.session == context.session + && users.insert(&b.identity.user) && participations.insert(&b.identity.participation) + && decode(&b.key)?.len() == 32)?; + } + Ok(roster) +} + +#[wasm_bindgen] +pub struct Session { + provider: OpenMlsRustCrypto, + signer: SignatureKeyPair, + credential: CredentialWithKey, + id: Identity, + host: String, + group: Option, +} +#[wasm_bindgen] +impl Session { + #[wasm_bindgen(constructor)] + pub fn new(identity_json: &str, host: &str) -> Result { + let id: Identity = serde_json::from_str(identity_json).map_err(err)?; + require([&id.channel, &id.session, &id.user, &id.participation].iter() + .all(|s| !s.is_empty() && s.len() <= 128) && !host.is_empty())?; + let provider = OpenMlsRustCrypto::default(); + let signer = SignatureKeyPair::new(SignatureScheme::ED25519).map_err(err)?; + signer.store(provider.storage()).map_err(err)?; + let credential = CredentialWithKey { + credential: BasicCredential::new(serde_json::to_vec(&id).map_err(err)?).into(), + signature_key: signer.public().into(), + }; + Ok(Self { provider, signer, credential, id, host: host.into(), group: None }) + } + pub fn public_key(&self) -> String { B64.encode(self.signer.public()) } + pub fn key_package(&self) -> Result { + require(self.group.is_none())?; + let kp = KeyPackage::builder().build(SUITE, &self.provider, &self.signer, self.credential.clone()).map_err(err)?; + encode(kp.key_package()) + } + pub fn create(&mut self) -> Result<(), String> { + require(self.group.is_none() && self.id.participation == self.host)?; + self.group = Some(MlsGroup::builder().ciphersuite(SUITE) + .use_ratchet_tree_extension(true) + .set_past_epoch_deletion_policy(PastEpochDeletionPolicy::MaxEpochs(0)) + .with_group_id(GroupId::from_slice(self.id.session.as_bytes())) + .build(&self.provider, &self.signer, self.credential.clone()).map_err(err)?); + Ok(()) + } + pub fn join(&mut self, welcome: &str, bindings: &str) -> Result<(), String> { + require(self.group.is_none() && self.id.participation != self.host)?; + let roster = roster(bindings, &self.id)?; + let msg = MlsMessageIn::tls_deserialize_exact(decode(welcome)?).map_err(err)?; + let welcome = match msg.extract() { MlsMessageBodyIn::Welcome(w) => w, _ => return Err(err(())) }; + let config = MlsGroupJoinConfig::builder().use_ratchet_tree_extension(true) + .set_past_epoch_deletion_policy(PastEpochDeletionPolicy::MaxEpochs(0)).build(); + let group = StagedWelcome::new_from_welcome(&self.provider, &config, welcome, None).map_err(err)? + .into_group(&self.provider).map_err(err)?; + require(group.group_id().as_slice() == self.id.session.as_bytes() && group.ciphersuite() == SUITE)?; + let mut users = std::collections::HashSet::new(); + let mut found_host = false; + for m in group.members() { + let id = check(&m.credential, &m.signature_key, &self.id, &roster)?; + require(users.insert(id.user))?; + found_host |= id.participation == self.host; + } + require(found_host)?; + self.group = Some(group); + Ok(()) + } + pub fn add(&mut self, key_package: &str, expected: &str, bindings: &str) -> Result { + require(self.id.participation == self.host)?; + let roster = roster(bindings, &self.id)?; + let kp = KeyPackageIn::tls_deserialize_exact(decode(key_package)?).map_err(err)? + .validate(self.provider.crypto(), ProtocolVersion::Mls10).map_err(err)?; + let id = check(kp.leaf_node().credential(), kp.leaf_node().signature_key().as_slice(), &self.id, &roster)?; + require(id.participation == expected)?; + let group = self.group.as_mut().ok_or_else(|| err(()))?; + require(group.members().count() < 128 && !group.members().any(|m| identity(&m.credential).is_ok_and(|i| i.user == id.user)))?; + let (commit, welcome, _) = group.add_members(&self.provider, &self.signer, &[kp]).map_err(err)?; + let result = json!({"commit": encode(&commit)?, "welcome": encode(&welcome)?}); + group.merge_pending_commit(&self.provider).map_err(err)?; + Ok(result.to_string()) + } + pub fn remove(&mut self, participation: &str, bindings: &str) -> Result { + require(self.id.participation == self.host && participation != self.host)?; + let roster = roster(bindings, &self.id)?; + require(!roster.iter().any(|b| b.identity.participation == participation))?; + let group = self.group.as_mut().ok_or_else(|| err(()))?; + let index = group.members().find(|m| identity(&m.credential).is_ok_and(|i| i.participation == participation)) + .ok_or_else(|| err(()))?.index; + let (commit, _, _) = group.remove_members(&self.provider, &self.signer, &[index]).map_err(err)?; + let out = encode(&commit)?; + group.merge_pending_commit(&self.provider).map_err(err)?; + Ok(out) + } + pub fn update(&mut self) -> Result { + let group = self.group.as_mut().ok_or_else(|| err(()))?; + if self.id.participation == self.host { + let bundle = group.self_update(&self.provider, &self.signer, LeafNodeParameters::default()).map_err(err)?; + let out = encode(bundle.commit())?; + group.merge_pending_commit(&self.provider).map_err(err)?; + Ok(out) + } else { + let (proposal, _) = group.propose_self_update(&self.provider, &self.signer, LeafNodeParameters::default()).map_err(err)?; + encode(&proposal) + } + } + pub fn commit_updates(&mut self) -> Result { + require(self.id.participation == self.host)?; + let group = self.group.as_mut().ok_or_else(|| err(()))?; + let (commit, _, _) = group.commit_to_pending_proposals(&self.provider, &self.signer).map_err(err)?; + let out = encode(&commit)?; + group.merge_pending_commit(&self.provider).map_err(err)?; + Ok(out) + } + pub fn encrypt(&mut self, text: &str) -> Result { + require(text.len() <= 100 * 1024)?; // Includes the authenticated receipt/message envelope. + let group = self.group.as_mut().ok_or_else(|| err(()))?; + encode(&group.create_message(&self.provider, &self.signer, text.as_bytes()).map_err(err)?) + } + pub fn process(&mut self, packet: &str, expected: &str, bindings: &str) -> Result { + let roster = roster(bindings, &self.id)?; + let group = self.group.as_mut().ok_or_else(|| err(()))?; + let msg = MlsMessageIn::tls_deserialize_exact(decode(packet)?).map_err(err)?; + let processed = group.process_message(&self.provider, msg.try_into_protocol_message().map_err(err)?).map_err(err)?; + let index = match processed.sender() { Sender::Member(i) => *i, _ => return Err(err(())) }; + let member = group.member_at(index).ok_or_else(|| err(()))?; + let sender = check(processed.credential(), &member.signature_key, &self.id, &roster)?; + require(sender.participation == expected)?; + let result = match processed.into_content() { + ProcessedMessageContent::ApplicationMessage(m) => { + let bytes = m.into_bytes(); + require(bytes.len() <= 100 * 1024)?; + json!({"kind":"application", "sender":sender, "text":String::from_utf8(bytes).map_err(err)?}) + }, + ProcessedMessageContent::ProposalMessage(p) => { + match p.proposal() { + Proposal::Update(u) => { + let id = check(u.leaf_node().credential(), u.leaf_node().signature_key().as_slice(), &self.id, &roster)?; + require(id == sender)?; + }, + _ => return Err(err(())), + } + group.store_pending_proposal(self.provider.storage(), *p).map_err(err)?; + json!({"kind":"proposal"}) + }, + ProcessedMessageContent::StagedCommitMessage(c) => { + require(sender.participation == self.host)?; + if let Some(leaf) = c.update_path_leaf_node() { + require(check(leaf.credential(), leaf.signature_key().as_slice(), &self.id, &roster)? == sender)?; + } + let mut added_users = std::collections::HashSet::new(); + for p in c.queued_proposals() { + match p.proposal() { + Proposal::Add(a) => { + let leaf = a.key_package().leaf_node(); + let id = check(leaf.credential(), leaf.signature_key().as_slice(), &self.id, &roster)?; + require(added_users.insert(id.user.clone()) && !group.members().any(|m| identity(&m.credential).is_ok_and(|i| i.user == id.user)))?; + }, + Proposal::Remove(r) => { + let m = group.member_at(r.removed()).ok_or_else(|| err(()))?; + let id = identity(&m.credential)?; + require(id.participation != self.host && !roster.iter().any(|b| b.identity == id))?; + }, + Proposal::Update(u) => { + let idx = match p.sender() { Sender::Member(i) => *i, _ => return Err(err(())) }; + let old = group.member_at(idx).ok_or_else(|| err(()))?; + require(check(u.leaf_node().credential(), u.leaf_node().signature_key().as_slice(), &self.id, &roster)? == identity(&old.credential)?)?; + }, + _ => return Err(err(())), + } + } + require(group.members().count() + c.add_proposals().count() - c.remove_proposals().count() <= 128)?; + group.merge_staged_commit(&self.provider, *c).map_err(err)?; + json!({"kind":"commit"}) + }, + _ => return Err(err(())), + }; + Ok(result.to_string()) + } + pub fn members(&self) -> Result { + let group = self.group.as_ref().ok_or_else(|| err(()))?; + let mut members = Vec::::new(); + for m in group.members() { + let id = identity(&m.credential)?; + let mut hash = Sha256::new(); + hash.update(b"Open WebUI temporary chat identity v1\0"); + hash.update(serde_json::to_vec(&id).map_err(err)?); + hash.update(&m.signature_key); + members.push(json!({"identity":id,"fingerprint":format!("{:x}",hash.finalize())})); + } + Ok(json!(members).to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn client(n: usize, session: &str) -> Session { + Session::new(&json!({"channel":"c","session":session,"user":format!("u{n}"),"participation":format!("p{n}")}).to_string(), "p0").unwrap() + } + fn bindings(clients: &[&Session]) -> String { + json!(clients.iter().map(|c| json!({"identity":c.id,"key":c.public_key()})).collect::>()).to_string() + } + #[test] + fn bindings_membership_and_epochs() { + let mut a = client(0,"s"); let mut b = client(1,"s"); let c = client(2,"other-session"); + a.create().unwrap(); + let roster = bindings(&[&a,&b]); + assert!(a.add(&c.key_package().unwrap(), "p2", &roster).is_err()); + let old = a.encrypt("past epoch").unwrap(); + let added: Value = serde_json::from_str(&a.add(&b.key_package().unwrap(), "p1", &roster).unwrap()).unwrap(); + b.join(added["welcome"].as_str().unwrap(), &roster).unwrap(); + assert!(b.process(&old,"p0",&roster).is_err()); + let msg = a.encrypt("hello").unwrap(); + assert!(b.process(&msg,"p0",&roster).unwrap().contains("hello")); + assert!(b.process(&msg,"p0",&roster).is_err()); + // A valid MLS commit from an ordinary member must still be rejected. + let bundle = b.group.as_mut().unwrap().self_update(&b.provider,&b.signer,LeafNodeParameters::default()).unwrap(); + assert!(a.process(&encode(bundle.commit()).unwrap(),"p1",&roster).is_err()); + assert!(b.remove("p0", &roster).is_err()); + } +} diff --git a/package-lock.json b/package-lock.json index fe39412956..efd8c0656e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "open-webui", - "version": "0.12.2", + "version": "0.12.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "open-webui", - "version": "0.12.2", + "version": "0.12.3", "dependencies": { "@azure/msal-browser": "^4.5.0", "@codemirror/lang-javascript": "^6.2.2", @@ -55,7 +55,7 @@ "codemirror-lang-hcl": "^0.1.0", "crc-32": "^1.2.2", "dayjs": "^1.11.10", - "docx-preview": "^0.4.0", + "docx-preview": "0.4.0", "dompurify": "^3.2.6", "eventsource-parser": "^1.1.2", "fast-deep-equal": "^3.1.3", diff --git a/package.json b/package.json index 910aa316aa..cce9df0b48 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,13 @@ { "name": "open-webui", - "version": "0.12.2", + "version": "0.12.3", "private": true, "scripts": { + "crypto:verify": "node scripts/encryption/artifact.mjs", + "predev": "npm run crypto:verify", + "predev:5050": "npm run crypto:verify", + "prebuild": "npm run crypto:verify", + "prebuild:watch": "npm run crypto:verify", "dev": "npm run pyodide:fetch && vite dev --host", "dev:5050": "npm run pyodide:fetch && vite dev --port 5050", "build": "npm run pyodide:fetch && vite build", @@ -100,7 +105,7 @@ "codemirror-lang-hcl": "^0.1.0", "crc-32": "^1.2.2", "dayjs": "^1.11.10", - "docx-preview": "^0.4.0", + "docx-preview": "0.4.0", "dompurify": "^3.2.6", "eventsource-parser": "^1.1.2", "fast-deep-equal": "^3.1.3", diff --git a/scripts/encryption/artifact.mjs b/scripts/encryption/artifact.mjs new file mode 100644 index 0000000000..d90c339e1c --- /dev/null +++ b/scripts/encryption/artifact.mjs @@ -0,0 +1,71 @@ +// Normal builds only read committed files. No Rust or network required. +import { createHash } from 'node:crypto'; +import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join, relative } from 'node:path'; +const root = new URL('../../', import.meta.url).pathname; +const source = 'crypto/encryption'; +const artifact = 'src/lib/encryption/vendor'; +const hash = (data) => createHash('sha256').update(data).digest('hex'); +function files(dir) { + return readdirSync(join(root, dir), { withFileTypes: true }) + .flatMap((e) => { + if (e.name === 'target' || e.name.startsWith('.')) return []; + const path = join(dir, e.name); + return e.isDirectory() ? files(path) : [path]; + }) + .sort(); +} +const inputs = [ + ...files(source), + 'scripts/encryption/build.py', + 'scripts/encryption/artifact.mjs' +].sort(); +const sourceFingerprint = hash( + inputs.map((p) => `${p}\0${hash(readFileSync(join(root, p)))}`).join('\n') +); +const manifestPath = join(root, artifact, 'manifest.json'); +if (process.argv.includes('--write')) { + const metadata = JSON.parse( + readFileSync(process.argv[process.argv.indexOf('--metadata') + 1], 'utf8') + ); + const checksums = Object.fromEntries( + files(artifact) + .filter((p) => !p.endsWith('/manifest.json')) + .map((p) => [relative(artifact, p), hash(readFileSync(join(root, p)))]) + ); + writeFileSync( + manifestPath, + JSON.stringify( + { + format: 1, + sourceFingerprint, + sourceRevision: process.env.CRYPTO_SOURCE_REVISION, + toolchain: '1.95.0', + wasmBindgen: '0.2.126', + dependencies: metadata.packages.map(({ name, version, license }) => ({ + name, + version, + license + })), + checksums + }, + null, + 2 + ) + '\n' + ); +} else { + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + if (manifest.sourceFingerprint !== sourceFingerprint) + throw Error('Encryption source changed: rebuild and review the artifact.'); + const actual = files(artifact) + .filter((p) => !p.endsWith('/manifest.json')) + .map((p) => relative(artifact, p)) + .sort(); + if (JSON.stringify(actual) !== JSON.stringify(Object.keys(manifest.checksums).sort())) + throw Error('Unexpected crypto artifact files'); + for (const [p, digest] of Object.entries(manifest.checksums)) { + if (hash(readFileSync(join(root, artifact, p))) !== digest) + throw Error(`Crypto artifact checksum mismatch: ${p}`); + } + console.log('Encryption artifact source and checksums verified'); +} diff --git a/scripts/encryption/build.py b/scripts/encryption/build.py new file mode 100644 index 0000000000..f6368c3db1 --- /dev/null +++ b/scripts/encryption/build.py @@ -0,0 +1,138 @@ +"""Manual artifact release only; never invoked by the application build.""" + +import json +import os +import subprocess +import tempfile +from functools import cache +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / 'crypto/encryption' +OUTPUT = ROOT / 'src/lib/encryption/vendor' + + +def run(*args, **kwargs): + return subprocess.run(args, cwd=SOURCE, check=True, **kwargs) + + +run('cargo', 'test', '--locked') +run('cargo', 'build', '--locked', '--release', '--target', 'wasm32-unknown-unknown') +run( + 'wasm-bindgen', + '--target', + 'web', + '--out-name', + 'encryption', + '--out-dir', + str(OUTPUT), + str(SOURCE / 'target/wasm32-unknown-unknown/release/encryption.wasm'), +) +# Use Cargo's actual target build graph, excluding tests and unused optional crates. +metadata = json.loads(run('cargo', 'metadata', '--locked', '--format-version', '1', capture_output=True).stdout) +tree = run( + 'cargo', + 'tree', + '--locked', + '--target', + 'wasm32-unknown-unknown', + '--edges', + 'normal,build', + '--prefix', + 'none', + '--format', + '{p}', + capture_output=True, +).stdout.decode() +used = {tuple(line.split()[:2]) for line in tree.splitlines()} +metadata['packages'] = [p for p in metadata['packages'] if (p['name'], 'v' + p['version']) in used] +assert {(p['name'], 'v' + p['version']) for p in metadata['packages']} == used + + +@cache +def download(url): + with urlopen(url, timeout=30) as response: + return response.read().decode('utf-8') + + +texts = {} +notices = [ + 'Third-Party Licenses', + '===================', + '', + 'Scope: Encryption WebAssembly artifact and its build-time dependencies.', + 'Packages are listed alphabetically with references to the full license and notice texts below.', + 'Identical texts are included once; package-specific copyright notices are preserved.', + 'Source code, including MPL-2.0 dependencies, is available at each Source URL below.', +] +for package in sorted(metadata['packages'], key=lambda p: (p['name'], p['version'])): + if package['source'] is None: + continue # The local wrapper is not a third-party dependency. + license = package['license'] or '' + choices = { + 'MIT', + 'Apache-2.0', + 'BSD-2-Clause', + 'BSD-3-Clause', + 'ISC', + 'Unicode-3.0', + 'Zlib', + 'CC0-1.0', + 'Unlicense', + 'MPL-2.0', + } + if not any(x.strip(' ()') in choices for x in license.replace('/', ' OR ').split(' OR ')): + raise RuntimeError(f'Review dependency license: {package["name"]} {license}') + directory = Path(package['manifest_path']).parent + documents = { + path.name: path.read_text() + for path in sorted(directory.iterdir()) + if path.is_file() and path.name.upper().startswith(('LICENSE', 'COPYING', 'NOTICE')) + } + if package.get('license_file'): + path = directory / package['license_file'] + documents[path.name] = path.read_text() + if not documents: + # Some published crates omit the workspace license; use their exact source revision. + if license == 'MPL-2.0': + url = 'https://www.mozilla.org/media/MPL/2.0/index.txt' + expected = 'Mozilla Public License Version 2.0' + else: + repository = urlparse(package['repository']) + if repository.netloc != 'github.com' or license not in ('MIT', 'Apache-2.0'): + raise RuntimeError(f'Missing license notice: {package["name"]}') + repo = '/'.join(repository.path.strip('/').split('/')[:2]) + revision = json.loads((directory / '.cargo_vcs_info.json').read_text())['git']['sha1'] + url = f'https://raw.githubusercontent.com/{repo}/{revision}/LICENSE' + expected = 'Permission is hereby granted' if license == 'MIT' else 'Apache License' + documents[url] = download(url) + if expected not in documents[url]: + raise RuntimeError(f'Unexpected license notice: {url}') + references = [] + for name, text in documents.items(): + text = text.strip() + if not text: + raise RuntimeError(f'Empty license notice: {package["name"]}/{name}') + number = texts.setdefault(text, len(texts) + 1) + references.append(f'{name}: [{number}]') + notices.append( + f'\nPackage: {package["name"]}\nVersion: {package["version"]}\nLicense: {license}\n' + f'Source: https://crates.io/api/v1/crates/{package["name"]}/{package["version"]}/download\n' + + '\n'.join(references) + ) +notices.extend( + f'\nLicense and notice text [{number}]\n--------------------------------\n{text}' for text, number in texts.items() +) +(OUTPUT / 'THIRD_PARTY_LICENSES.txt').write_text('\n'.join(notices) + '\n') +with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as temp: + json.dump(metadata, temp) + temp.flush() + env = dict( + os.environ, + CRYPTO_SOURCE_REVISION=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT).decode().strip(), + ) + subprocess.run( + ['node', str(ROOT / 'scripts/encryption/artifact.mjs'), '--write', '--metadata', temp.name], check=True, env=env + ) diff --git a/src/app.css b/src/app.css index 9228db7c6b..d5a7f782ae 100644 --- a/src/app.css +++ b/src/app.css @@ -117,7 +117,19 @@ textarea::-webkit-scrollbar-corner { } .markdown-prose { - @apply prose prose-sm dark:prose-invert max-w-none break-words !text-[0.9375rem] font-normal leading-relaxed prose-p:mt-0 prose-p:mb-2 prose-p:font-normal prose-p:leading-relaxed prose-headings:mt-2 prose-headings:mb-1 prose-headings:font-medium prose-headings:leading-snug prose-h1:text-xl prose-h2:text-lg prose-h3:text-base prose-strong:font-medium prose-code:before:content-none prose-code:after:content-none prose-ul:my-2 prose-ol:my-2 prose-li:my-0.5 prose-li:font-normal prose-pre:my-3 prose-table:my-0 prose-blockquote:my-3 prose-blockquote:font-normal prose-hr:my-4 prose-hr:border-gray-50 prose-hr:dark:border-gray-850/30 prose-img:my-2 [&>:first-child]:mt-0 [&>:last-child]:mb-0 [&_p:first-child]:mt-0 [&_h1:first-child]:mt-0 [&_h2:first-child]:mt-0 [&_h3:first-child]:mt-0 [&_h4:first-child]:mt-0 [&_h5:first-child]:mt-0 [&_h6:first-child]:mt-0; + @apply prose prose-sm dark:prose-invert max-w-none break-words !text-[0.9375rem] font-normal leading-relaxed prose-p:mt-0 prose-p:mb-2 prose-p:font-normal prose-p:leading-relaxed prose-headings:mt-2 prose-headings:mb-1 prose-headings:font-medium prose-headings:leading-snug prose-h1:text-xl prose-h2:text-lg prose-h3:text-base prose-strong:font-medium prose-code:before:content-none prose-code:after:content-none prose-ul:my-2 prose-ol:my-2 prose-li:my-0.5 prose-li:font-normal prose-pre:my-3 prose-table:my-0 prose-blockquote:text-[0.8125rem] prose-hr:my-4 prose-hr:border-gray-50 prose-hr:dark:border-gray-850/30 prose-img:my-2 [&>:first-child]:mt-0 [&>:last-child]:mb-0 [&_p:first-child]:mt-0 [&_h1:first-child]:mt-0 [&_h2:first-child]:mt-0 [&_h3:first-child]:mt-0 [&_h4:first-child]:mt-0 [&_h5:first-child]:mt-0 [&_h6:first-child]:mt-0; +} + +.markdown-prose, +.document-markdown { + @apply prose-blockquote:my-2 prose-blockquote:border-s-2 prose-blockquote:border-gray-200 prose-blockquote:dark:border-gray-700 prose-blockquote:ps-3 prose-blockquote:pe-0 prose-blockquote:leading-relaxed prose-blockquote:not-italic prose-blockquote:font-normal prose-blockquote:text-gray-600 prose-blockquote:dark:text-gray-400 prose-blockquote:[&>:first-child]:mt-0 prose-blockquote:[&>:last-child]:mb-0; +} + +.markdown-prose :where(blockquote p:first-of-type)::before, +.markdown-prose :where(blockquote p:last-of-type)::after, +.document-markdown :where(blockquote p:first-of-type)::before, +.document-markdown :where(blockquote p:last-of-type)::after { + content: none; } .markdown-prose-sm { @@ -1023,3 +1035,102 @@ html.high-contrast:not(.dark) .shimmer { color: var(--color-gray-800); -webkit-text-fill-color: var(--color-gray-800); } + +/* Shared by file previews and the attachment modal. */ +.office-preview { + font-size: 0.875rem; + line-height: 1.6; + color: var(--color-gray-800); + background: #fff; + border-radius: 0.25rem; +} +.dark .office-preview { + color: var(--color-gray-200); + background: var(--color-gray-900); +} +.office-preview table { + border-collapse: collapse; + font-size: 0.75rem; + font-family: ui-monospace, SFMono-Regular, 'SF Mono', Menlo, monospace; + line-height: 1.3; +} +.office-preview table td, +.office-preview table th { + border: 1px solid var(--color-gray-200); + padding: 0.25rem 0.625rem; + text-align: left; + white-space: nowrap; + user-select: text; + cursor: cell; + max-width: 18.75rem; + overflow: hidden; + text-overflow: ellipsis; +} +.dark .office-preview table td, +.dark .office-preview table th { + border-color: var(--color-gray-800); +} +/* Column letter headers */ +.office-preview table th.excel-col-hdr { + position: sticky; + top: 0; + z-index: 2; + background: var(--color-gray-100); + color: var(--color-gray-600); + font-weight: 500; + font-size: 0.65rem; + text-align: center; + padding: 0.1875rem 0.625rem; + border-bottom: 2px solid var(--color-gray-300); +} +.dark .office-preview table th.excel-col-hdr { + background: var(--color-gray-850); + color: var(--color-gray-400); + border-bottom-color: var(--color-gray-700); +} +/* Row number cells */ +.office-preview .excel-row-num { + position: sticky; + left: 0; + z-index: 1; + background: var(--color-gray-100); + color: var(--color-gray-600); + font-size: 0.6rem; + text-align: right !important; + padding: 0.25rem 0.5rem 0.25rem 0.25rem !important; + user-select: none; + width: 1px; + white-space: nowrap; + border-right: 2px solid var(--color-gray-300) !important; +} +.dark .office-preview .excel-row-num { + background: var(--color-gray-850); + color: var(--color-gray-400); + border-right-color: var(--color-gray-700) !important; +} +/* Corner cell (intersection of row nums and col headers) */ +.office-preview thead .excel-row-num { + z-index: 3; +} +/* Number cells right-aligned */ +.office-preview .excel-num { + text-align: right; + font-variant-numeric: tabular-nums; +} +/* Row hover and selection */ +.office-preview table tbody tr:nth-child(even) td:not(.excel-row-num) { + background: rgba(0, 0, 0, 0.015); +} +.dark .office-preview table tbody tr:nth-child(even) td:not(.excel-row-num) { + background: rgba(255, 255, 255, 0.02); +} +.office-preview table tbody tr:hover td:not(.excel-row-num) { + background: rgba(59, 130, 246, 0.06); +} +.dark .office-preview table tbody tr:hover td:not(.excel-row-num) { + background: rgba(59, 130, 246, 0.1); +} +.office-preview table td:focus { + outline: 2px solid rgba(59, 130, 246, 0.5); + outline-offset: -2px; +} diff --git a/src/lib/apis/channels/index.ts b/src/lib/apis/channels/index.ts index 0d040e4395..a3dd5e840b 100644 --- a/src/lib/apis/channels/index.ts +++ b/src/lib/apis/channels/index.ts @@ -479,12 +479,13 @@ export const getChannelThreadMessages = async ( channel_id: string, message_id: string, skip: number = 0, - limit: number = 50 + limit: number = 50, + anchor: { around?: string; before?: string } = {} ) => { let error = null; const res = await fetch( - `${WEBUI_API_BASE_URL}/channels/${channel_id}/messages/${message_id}/thread?skip=${skip}&limit=${limit}`, + `${WEBUI_API_BASE_URL}/channels/${channel_id}/messages/${message_id}/thread?skip=${skip}&limit=${limit}&${new URLSearchParams(anchor)}`, { method: 'GET', headers: { diff --git a/src/lib/apis/search/index.ts b/src/lib/apis/search/index.ts new file mode 100644 index 0000000000..9f967b4552 --- /dev/null +++ b/src/lib/apis/search/index.ts @@ -0,0 +1,36 @@ +import { WEBUI_API_BASE_URL } from '$lib/constants'; + +export type SearchKind = 'chat' | 'message' | 'note'; +export type SearchResult = { + id: string; + kind: SearchKind; + title: string; + context: string; + snippet: string; + preview: string; + url: string; + updated_at: number; + shared: boolean; + pinned: boolean; +}; +export type SearchScope = { id: string; name: string; kind: 'folder' | 'channel' }; +export type SearchResponse = { + items: SearchResult[]; + has_more: boolean; + scopes: SearchScope[]; + kinds: SearchKind[]; +}; + +export const searchWorkspace = async ( + token: string, + filters: { q: string; kind: string; sharing: string; scope: string; page: number }, + signal: AbortSignal +): Promise => { + const params = new URLSearchParams({ ...filters, page: String(filters.page) }); + const response = await fetch(`${WEBUI_API_BASE_URL}/search?${params}`, { + headers: { Authorization: `Bearer ${token}` }, + signal + }); + if (!response.ok) throw new Error('Search is unavailable. Please try again.'); + return response.json(); +}; diff --git a/src/lib/apis/terminal/index.ts b/src/lib/apis/terminal/index.ts index acea145f29..4dca975669 100644 --- a/src/lib/apis/terminal/index.ts +++ b/src/lib/apis/terminal/index.ts @@ -1,6 +1,36 @@ import { get } from 'svelte/store'; import { config } from '$lib/stores'; import { checkSourceSize, readSourceResponse } from './sourceText'; +// The same attachment is used by @mentions, terminal uploads, and Files-pane requests. +export type FilesystemFile = { + type: 'filesystem'; + id: string; + path: string; + url: string; + name: string; + status: string; + size?: number; + modified?: number; + terminal_id?: string; + session_id?: string | null; +}; +export const filesystemFile = ( + entry: { path: string; name: string; size?: number; modified?: number }, + terminalId: string, + sessionId: string | null +): FilesystemFile => ({ + type: 'filesystem', + id: entry.path, + path: entry.path, + url: entry.path, + name: entry.name, + size: entry.size, + modified: entry.modified, + status: 'processed', + terminal_id: terminalId, + session_id: sessionId +}); + export type FileEntry = { name: string; type: 'file' | 'directory'; @@ -396,12 +426,13 @@ export const downloadFileBlob = async ( baseUrl: string, apiKey: string, path: string, - sessionId?: string + sessionId?: string, + signal?: AbortSignal ): Promise<{ blob: Blob; filename: string } | null> => { const url = `${baseUrl.replace(/\/$/, '')}/files/view?path=${encodeURIComponent(path)}`; const headers: Record = bearerHeaders(apiKey); if (sessionId) headers['X-Session-Id'] = sessionId; - const res = await fetch(url, { headers }).catch(() => null); + const res = await fetch(url, { headers, signal }).catch(() => null); if (!res || !res.ok) return null; @@ -415,12 +446,13 @@ export const downloadFilePreview = async ( baseUrl: string, apiKey: string, path: string, - sessionId?: string + sessionId?: string, + signal?: AbortSignal ): Promise<{ blob: Blob; filename: string } | null> => { const url = `${baseUrl.replace(/\/$/, '')}/files/view?path=${encodeURIComponent(path)}&preview=true`; const headers: Record = bearerHeaders(apiKey); if (sessionId) headers['X-Session-Id'] = sessionId; - const res = await fetch(url, { headers }).catch(() => null); + const res = await fetch(url, { headers, signal }).catch(() => null); if (!res) return null; if (!res.ok) return null; diff --git a/src/lib/components/admin/Settings/General.svelte b/src/lib/components/admin/Settings/General.svelte index 24e865fbea..db9c9ea478 100644 --- a/src/lib/components/admin/Settings/General.svelte +++ b/src/lib/components/admin/Settings/General.svelte @@ -84,6 +84,8 @@ const cleaned = entriesToI18n(uiI18nEntries); const res = await updateAdminConfig(localStorage.token, { ...adminConfig, + SKILL_CONTENT_MAX_CHARS: adminConfig.SKILL_CONTENT_MAX_CHARS ?? null, + SKILL_PREVIEW_MAX_CHARS: adminConfig.SKILL_PREVIEW_MAX_CHARS ?? null, DEFAULT_INTERFACE_SETTINGS: defaultInterfaceSettings, I18N: cleaned }); @@ -337,6 +339,36 @@ /> {/if} + + + + + + {#if adminConfig.ENABLE_CHANNELS} + + + = {}; + export let defaultPermissions: Record = {}; // Reactive statement to ensure all fields are present in `permissions` $: { @@ -1007,6 +1007,25 @@ {/if} +
+
+
+ {$i18n.t('Encrypted Channels')} +
+ +
+ {#if defaultPermissions?.features?.encrypted_channels && !permissions.features.encrypted_channels} +
+
+ {$i18n.t('This is a default user permission and will remain enabled.')} +
+
+ {/if} +
+
diff --git a/src/lib/components/channel/Channel.svelte b/src/lib/components/channel/Channel.svelte index 23095a5577..a5bccf4aaa 100644 --- a/src/lib/components/channel/Channel.svelte +++ b/src/lib/components/channel/Channel.svelte @@ -5,8 +5,10 @@ import { goto } from '$app/navigation'; import { page } from '$app/stores'; import { v4 as uuidv4 } from 'uuid'; + import type { Socket } from 'socket.io-client'; import { + config, chatId, channels, channelId as _channelId, @@ -19,6 +21,7 @@ import Messages from './Messages.svelte'; import MessageInput from './MessageInput.svelte'; import Navbar from './Navbar.svelte'; + import { canUseEncryption, connected, enter, request } from '$lib/encryption/client'; import Drawer from '../common/Drawer.svelte'; import ResizableSidePanel from '../common/ResizableSidePanel.svelte'; import EllipsisVertical from '../icons/EllipsisVertical.svelte'; @@ -28,6 +31,58 @@ export let id = ''; + let mounted = false; + let invitationSession: string | null = null; + let stopInvitation = () => {}; + $: watchInvitation( + $socket, + channel?.id === id ? id : null, + mounted && canUseEncryption($user, $config?.features) + ); + + function watchInvitation( + connection: Socket | null, + channelId: string | null, + enabled: boolean | undefined + ) { + stopInvitation(); + stopInvitation = () => {}; + invitationSession = null; + if (!enabled || !connection || !channelId) return; + let alive = true; + let revision = 0; + const receive = (event: { channel: string; session: string | null }) => { + if (event.channel === channelId) { + revision++; + invitationSession = event.session; + } + }; + const clear = () => { + revision++; + invitationSession = null; + }; + const query = () => { + if (!connected(connection)) return; + const issued = revision; + void request(connection, 'status', { channel: channelId }) + .then((result) => { + if (alive && revision === issued) + invitationSession = result.available ? result.session : null; + }) + .catch(() => {}); + }; + connection.on('encryption:available', receive); + connection.on('disconnect', clear); + connection.on('connect', query); + query(); + stopInvitation = () => { + alive = false; + connection.off('encryption:available', receive); + connection.off('disconnect', clear); + connection.off('connect', query); + }; + } + let currentId = null; let scrollEnd = true; @@ -51,12 +106,15 @@ $: if (channel && $page.url.searchParams.get('thread')) { threadId = $page.url.searchParams.get('thread'); - window.history.replaceState(history.state, '', `/channels/${id}`); + if (!$page.url.searchParams.has('message')) { + window.history.replaceState(history.state, '', `/channels/${id}`); + } } const scrollToBottom = () => { if (messagesContainerElement) { - messagesContainerElement.scrollTop = messagesContainerElement.scrollHeight; + // The channel uses flex-col-reverse, whose bottom is scrollTop = 0. + messagesContainerElement.scrollTop = 0; } }; @@ -147,18 +205,29 @@ if (type === 'message') { if ((data?.parent_id ?? null) === null) { - const tempId = data?.temp_id ?? null; - messages = [ - { ...data, temp_id: null }, - ...messages.filter((m) => !tempId || m?.temp_id !== tempId) - ]; + const idx = (messages ?? []).findIndex( + (message) => + message.id === data.id || (data.temp_id && message.temp_id === data.temp_id) + ); + if (idx === -1) { + messages = [{ ...data, temp_id: null }, ...(messages ?? [])]; + } else { + // Confirm the existing row without changing its position or render key. + const previous = messages[idx]; + messages[idx] = { + ...previous, + ...data, + temp_id: null, + client_id: previous.client_id ?? previous.id + }; + } if (typingUsers.find((user) => user.id === event.user.id)) { typingUsers = typingUsers.filter((user) => user.id !== event.user.id); } await tick(); - if (scrollEnd) { + if (idx === -1 && scrollEnd) { scrollToBottom(); } } @@ -166,7 +235,7 @@ const idx = messages.findIndex((message) => message.id === data.id); if (idx !== -1) { - messages[idx] = data; + messages[idx] = { ...messages[idx], ...data }; } } else if (type === 'message:delete') { messages = messages @@ -188,12 +257,12 @@ const idx = messages.findIndex((message) => message.id === data.id); if (idx !== -1) { - messages[idx] = data; + messages[idx] = { ...messages[idx], ...data }; } } else if (type.includes('message:reaction')) { const idx = messages.findIndex((message) => message.id === data.id); if (idx !== -1) { - messages[idx] = data; + messages[idx] = { ...messages[idx], ...data }; } } else if (type === 'typing' && event.message_id === null) { if (event.user.id === $user?.id) { @@ -254,6 +323,7 @@ { ...message, id: tempId, + client_id: tempId, user_id: $user?.id, user: $user, reply_to_message: reply_to_message ?? null, @@ -263,14 +333,13 @@ ...messages ]; - const res = await sendMessage(localStorage.token, channel_id, message).catch((error) => { + await tick(); + if (id === channel_id) scrollToBottom(); + + await sendMessage(localStorage.token, channel_id, message).catch((error) => { toast.error(`${error}`); return null; }); - - if (res && id === channel_id && messagesContainerElement) { - messagesContainerElement.scrollTop = messagesContainerElement.scrollHeight; - } }; const onChange = async () => { @@ -297,6 +366,7 @@ }; onMount(() => { + mounted = true; if ($chatId) { chatId.set(''); } @@ -310,6 +380,7 @@ }); onDestroy(() => { + stopInvitation(); // last read at updateLastReadAt(id); _channelId.set(null); @@ -358,7 +429,30 @@ return null; }); }} - /> + > + + {#if invitationSession} +
+ {$i18n.t('Encrypted chat in progress')} + + +
+ {/if} + + {#if channel && messages !== null}
diff --git a/src/lib/components/channel/ChannelInfoModal.svelte b/src/lib/components/channel/ChannelInfoModal.svelte index c6c703fa48..9ebcbde39b 100644 --- a/src/lib/components/channel/ChannelInfoModal.svelte +++ b/src/lib/components/channel/ChannelInfoModal.svelte @@ -97,8 +97,8 @@ onUpdate={updateMembers} /> -
-
+
+
{#if channel?.type === 'dm'} @@ -130,16 +130,18 @@
-
-
+
+
{ e.preventDefault(); submitHandler(); }} > -
+
-
+
{#if users === null || total === null}
{:else} -
+
{$i18n.t('Members')} @@ -136,7 +136,7 @@ {#if search} -
+
@@ -165,7 +165,9 @@ {/if} {#if users.length > 0 || filteredApps.length > 0} -
+
{ closeDirectoryMenu(); @@ -1936,58 +2281,64 @@ }} /> {:else if selectedFile !== null} - { - if (!excelWorkbook) return; - selectedExcelSheet = sheet; - const { excelToTable } = await import('$lib/utils/excelToTable'); - const result = await excelToTable(excelWorkbook.Sheets[sheet]); - const DOMPurify = (await import('dompurify')).default; - fileOfficeHtml = DOMPurify.sanitize(result.html); - }} - baseUrl={previewBaseUrl} - apiKey={selectedTerminal?.key ?? ''} - overlay={overlay || isDraggingHandle} - onSave={async (content) => { - const terminal = selectedTerminal; - if (!terminal || !selectedFile || !selectedFileWritable) return; - const fileName = selectedFile.split('/').pop() ?? 'file'; - const dir = selectedFile.substring(0, selectedFile.lastIndexOf('/') + 1) || '/'; - const file = new File([content], fileName, { type: 'text/plain' }); - const result = await uploadToTerminal( - terminal.url, - terminal.key, - dir, - file, - chatId ?? undefined - ); - toast[result ? 'success' : 'error']( - result ? $i18n.t('File saved') : $i18n.t('Failed to save file') - ); - if (result) fileContent = content; - }} - /> + {#if supportsComments(selectedFile) && selectedTerminal} + {#key `${selectedTerminal.url}|${selectedTerminal.key}|${chatId}|${selectedFile}|${commentGeneration}`} + { + openedFile = pendingFile; + publishFile(); + }} + bind:currentSlide + bind:sheet={selectedExcelSheet} + /> + {/key} + {:else} + showFileNavPath.set({ path })} + bind:currentSlide + {selectedFile} + readOnly={!selectedFileWritable} + {fileLoading} + {fileImageUrl} + {fileVideoUrl} + {fileAudioUrl} + {filePdfData} + {fileSqliteData} + {fileDocxData} + {fileContent} + {fileOfficeHtml} + {fileOfficeSlides} + targetPage={documentTargetPage} + {excelSheetNames} + {selectedExcelSheet} + searchTarget={fileSearchTarget} + onSheetChange={async (sheet) => { + if (!excelWorkbook) return; + selectedExcelSheet = sheet; + const { excelToTable } = await import('$lib/utils/excelToTable'); + const result = await excelToTable(excelWorkbook.Sheets[sheet]); + const DOMPurify = (await import('dompurify')).default; + fileOfficeHtml = DOMPurify.sanitize(result.html); + }} + baseUrl={previewBaseUrl} + apiKey={selectedTerminal?.key ?? ''} + overlay={overlay || isDraggingHandle} + onSave={saveFile} + /> + {/if} {:else} {#if isSearching} {#if matchLoading} @@ -2126,13 +2477,14 @@ {#if !isSearching && !loading && !error && !uploading && !($selectedTerminalId && $terminalServers === null)} {#if creatingFolder} -
+
{ if (e.key === 'Enter') submitNewFolder(); if (e.key === 'Escape') { @@ -2145,13 +2497,14 @@
{/if} {#if creatingFile} -
+
{ if (e.key === 'Enter') submitNewFile(); if (e.key === 'Escape') { @@ -2289,26 +2642,26 @@