refac

backend/open_webui/tools/builtin.py

@@ -69,7 +69,7 @@ from open_webui.utils.chat_id import is_saved_chat_id
 from open_webui.utils.json_codec import JSONCodec
 from open_webui.utils.notifications import notify_target
 from open_webui.utils.sanitize import sanitize_code
-from open_webui.utils.skill_files import SkillFile, SkillFileOperation, bounded_skill_manifest, skill_content_page
+from open_webui.utils.skill_files import SkillFile, SkillFileOperation, skill_content_page, skill_overview
 
 log = logging.getLogger(__name__)
 
@@ -3506,7 +3506,9 @@ async def view_skill(
     __metadata__: dict = None,
     __event_call__: callable = None,
 ) -> str:
-    """Read skill instructions and file list. Use read_skill_file to continue from next_offset.
+    """Read skill instructions and file list. Large skills return an opening preview and section offsets.
+
+    Use read_skill_file to read applicable sections, including mandatory rules, when the instructions are incomplete.
 
     :param id: Skill ID from the available skills manifest.
     """
@@ -3519,7 +3521,12 @@ async def view_skill(
 
             skill_name = unquote(id.removeprefix(terminal_skill_prefix))
             skill = await get_terminal_skill(
-                __request__, __user__, __metadata__ or {}, skill_name, {'__event_call__': __event_call__}
+                __request__,
+                __user__,
+                __metadata__ if __metadata__ is not None else {},
+                skill_name,
+                {'__event_call__': __event_call__},
+                overview=True,
             )
             if not skill:
                 return JSONCodec.dumps({'error': f"Skill '{id}' not found"})
@@ -3545,8 +3552,7 @@ async def view_skill(
             {
                 'id': skill.id,
                 'name': snapshot['name'],
-                **skill_content_page(snapshot['content']),
-                **bounded_skill_manifest(file_summaries(snapshot['data']['files'])),
+                **skill_overview(snapshot['content'], file_summaries(snapshot['data']['files'])),
             },
             ensure_ascii=False,
         )
@@ -3569,7 +3575,7 @@ async def read_skill_file(
     :param id: Skill ID.
     :param path: Relative path within the skill.
     :param offset: Character offset for paging text.
-    :param max_chars: Maximum characters to return, up to 100000.
+    :param max_chars: Maximum characters to return, up to 20000 (default 10000).
     """
     try:
         from types import SimpleNamespace

backend/open_webui/utils/middleware.py

@@ -2872,7 +2872,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
 
     if skill_ids or (use_builtin_tools and model_builtin_tools.get('skills', True)):
         from open_webui.models.skills import Skills as SkillsModel
-        from open_webui.utils.skill_files import bounded_skill_manifest, format_skill_content, skill_content_page
+        from open_webui.utils.skill_files import format_skill_content, skill_overview
         from open_webui.utils.terminals import (
             format_terminal_skill_context,
             format_terminal_skill_manifest_entry,
@@ -2909,13 +2909,13 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                     continue
                 skill_versions[skill.id] = version_id
                 skill_tools_enabled = use_builtin_tools and model_builtin_tools.get('skills', True)
-                root_content = format_skill_content(
-                    skill_content_page(snapshot['content']), skill.id, skill_tools_enabled
+                overview = skill_overview(
+                    snapshot['content'],
+                    [f['path'] for f in snapshot['data']['files'] if f['path'] != 'SKILL.md'],
+                    tools_enabled=skill_tools_enabled,
                 )
-                manifest = bounded_skill_manifest(
-                    [f['path'] for f in snapshot['data']['files'] if f['path'] != 'SKILL.md']
-                )
-                resources = '\n'.join(manifest['files'])
+                root_content = format_skill_content(overview, skill.id, skill_tools_enabled)
+                resources = '\n'.join(overview['files'])
                 resource_hint = (
                     f'\nRead supporting files with read_skill_file(id="{skill.id}", path=...).\n{resources}'
                     if use_builtin_tools and model_builtin_tools.get('skills', True)
@@ -2923,8 +2923,8 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                     if resources
                     else ''
                 )
-                if manifest.get('notice'):
-                    resource_hint += '\n' + manifest['notice']
+                if overview.get('notice'):
+                    resource_hint += '\n' + overview['notice']
                 form_data['messages'] = add_or_update_system_message(
                     f'<skill id="{skill.id}" name="{skill.name}">\n'
                     f'{root_content}{resource_hint}\n</skill>',
@@ -2956,7 +2956,15 @@ async def process_chat_payload(request, form_data, user, metadata, model):
             sid = skill['id']
             if sid in mentioned_skill_ids or not use_builtin_tools or not model_builtin_tools.get('skills', True):
                 skill_name = unquote(sid.removeprefix(terminal_skill_prefix))
-                loaded = await get_terminal_skill(request, user.model_dump(), metadata, skill_name, extra_params)
+                loaded = await get_terminal_skill(
+                    request,
+                    user.model_dump(),
+                    metadata,
+                    skill_name,
+                    extra_params,
+                    overview=True,
+                    tools_enabled=use_builtin_tools and model_builtin_tools.get('skills', True),
+                )
                 if loaded:
                     form_data['messages'] = add_or_update_system_message(
                         format_terminal_skill_context(

backend/open_webui/utils/skill_files.py

@@ -19,7 +19,10 @@ MAX_FILE_BYTES = 10 * 1024 * 1024
 MAX_SKILL_BYTES = 50 * 1024 * 1024
 MAX_IMPORT_BYTES = 200 * 1024 * 1024
 MAX_FILES = 1000
-SKILL_CONTENT_MAX_CHARS = 100_000
+SKILL_CONTENT_MAX_CHARS = 20_000
+SKILL_INJECTION_MAX_CHARS = 100_000
+SKILL_PREVIEW_MAX_CHARS = 5_000
+SKILL_OVERVIEW_LIST_MAX_CHARS = 2_500
 SKILL_MANIFEST_MAX_ENTRIES = 50
 SKILL_MANIFEST_MAX_CHARS = 5_000
 
@@ -30,11 +33,11 @@ def skill_content_page(content: str, offset: int = 0, max_chars: int = SKILL_CON
     return {'content': content[offset:end], 'next_offset': end if end < len(content) else None}
 
 
-def bounded_skill_manifest(entries: list, field: str = 'files') -> dict:
+def bounded_skill_manifest(entries: list, field: str = 'files', max_chars: int = SKILL_MANIFEST_MAX_CHARS) -> dict:
     bounded, size = [], 2  # Include the JSON array brackets and separators in the budget.
     for entry in entries[:SKILL_MANIFEST_MAX_ENTRIES]:
         entry_size = len(json.dumps(entry, ensure_ascii=False)) + (2 if bounded else 0)
-        if size + entry_size > SKILL_MANIFEST_MAX_CHARS:
+        if size + entry_size > max_chars:
             break
         bounded.append(entry)
         size += entry_size
@@ -44,13 +47,73 @@ def bounded_skill_manifest(entries: list, field: str = 'files') -> dict:
     return result
 
 
+def skill_heading_index(content: str) -> dict:
+    headings, size, offset = [], 2, 0
+    frontmatter_match = re.match(r'\A---[ \t]*\r?\n.*?\r?\n---[ \t]*(?:\r?\n|$)', content, re.DOTALL)
+    body_start = frontmatter_match.end() if frontmatter_match else 0
+    fence = None
+    for line in content.splitlines(keepends=True):
+        line_offset = offset
+        offset += len(line)
+        if line_offset < body_start:
+            continue
+        text = line.rstrip('\r\n')
+        marker = re.match(r' {0,3}(`{3,}|~{3,})(.*)$', text)
+        if fence:
+            if marker and marker[1][0] == fence[0] and len(marker[1]) >= len(fence) and not marker[2].strip():
+                fence = None
+            continue
+        if marker and (marker[1][0] == '~' or '`' not in marker[2]):
+            fence = marker[1]
+            continue
+        heading = re.match(r' {0,3}(#{1,6})(?:[ \t]+(.*)|$)', text)
+        if heading:
+            title = re.sub(r'(?:^|[ \t]+)#+[ \t]*$', '', heading[2] or '').strip()
+            entry = {'title': title, 'level': len(heading[1]), 'offset': line_offset}
+            entry_size = len(json.dumps(entry, ensure_ascii=False)) + (2 if headings else 0)
+            if size + entry_size > SKILL_OVERVIEW_LIST_MAX_CHARS:
+                return {'headings': headings, 'headings_truncated': True}
+            headings.append(entry)
+            size += entry_size
+    return {'headings': headings, 'headings_truncated': False}
+
+
+def skill_overview(content: str, files: list, field: str = 'files', tools_enabled: bool = True) -> dict:
+    if not tools_enabled:
+        return {
+            'content': content[:SKILL_INJECTION_MAX_CHARS],
+            'next_offset': SKILL_INJECTION_MAX_CHARS if len(content) > SKILL_INJECTION_MAX_CHARS else None,
+            **bounded_skill_manifest(files, field),
+        }
+    large = len(content) > SKILL_CONTENT_MAX_CHARS
+    overview = {
+        **skill_content_page(content, max_chars=SKILL_PREVIEW_MAX_CHARS if large else SKILL_CONTENT_MAX_CHARS),
+        **bounded_skill_manifest(files, field, SKILL_OVERVIEW_LIST_MAX_CHARS),
+    }
+    if large:
+        overview.update(skill_heading_index(content))
+        overview['reading_hint'] = (
+            'This is an opening preview, not the complete instructions. Read applicable sections, including mandatory '
+            'rules, with read_skill_file using the skill id, path="SKILL.md", a heading offset, and max_chars up to 20000. '
+            'Offsets count characters in the original file. Use next_offset for additional text when needed.'
+        )
+    return overview
+
+
 def format_skill_content(page: dict, skill_id: str, tools_enabled: bool) -> str:
     content = page['content']
+    if tools_enabled and 'headings' in page:
+        content += '\nSection index: ' + json.dumps(page['headings'], ensure_ascii=False)
+        if page.get('headings_truncated'):
+            content += '\nAdditional headings omitted.'
+        content += f'\n{page["reading_hint"]} Skill ID: {json.dumps(skill_id)}.'
+        content += f'\nnext_offset: {page["next_offset"]}'
+        return content
     if page['next_offset'] is not None:
         content += '\nSkill instructions truncated.'
         if tools_enabled:
             content += (
-                f'\nContinue reading with read_skill_file(id={json.dumps(skill_id)}, '
+                f'\nRead additional text when needed with read_skill_file(id={json.dumps(skill_id)}, '
                 f'path="SKILL.md", offset={page["next_offset"]}).'
             )
     return content

backend/open_webui/utils/terminals.py

@@ -14,6 +14,7 @@ from open_webui.utils.skill_files import (
     bounded_skill_manifest,
     format_skill_content,
     skill_content_page,
+    skill_overview,
 )
 
 TERMINAL_CONTEXT_HEADER = 'X-Terminal-Context-Id'
@@ -242,6 +243,8 @@ async def get_terminal_skill(
     offset: int = 0,
     max_chars: int = SKILL_CONTENT_MAX_CHARS,
     refresh: bool = True,
+    overview: bool = False,
+    tools_enabled: bool = True,
 ) -> dict | None:
     skill = await get_terminal_json(
         request, user, metadata, f'/skills/read?name={quote(skill_name, safe="")}', extra_params
@@ -266,9 +269,15 @@ async def get_terminal_skill(
     return {
         'name': skill.get('name'),
         'description': skill.get('description'),
-        **skill_content_page(skill['content'], offset, max_chars),
         'directory': directory,
-        **bounded_skill_manifest(resources, 'resources'),
+        **(
+            skill_overview(skill['content'], resources, 'resources', tools_enabled)
+            if overview
+            else {
+                **skill_content_page(skill['content'], offset, max_chars),
+                **bounded_skill_manifest(resources, 'resources'),
+            }
+        ),
     }